15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Copyright (c) 2011, Google Inc. 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// All rights reserved. 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Redistribution and use in source and binary forms, with or without 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// modification, are permitted provided that the following conditions are 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// met: 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * Redistributions of source code must retain the above copyright 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// notice, this list of conditions and the following disclaimer. 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * Redistributions in binary form must reproduce the above 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// copyright notice, this list of conditions and the following disclaimer 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// in the documentation and/or other materials provided with the 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// distribution. 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * Neither the name of Google Inc. nor the names of its 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// contributors may be used to endorse or promote products derived from 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// this software without specific prior written permission. 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// --- 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Author: Sasha Levitskiy 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// based on atomicops-internals by Sanjay Ghemawat 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This file is an internal atomic implementation, use base/atomicops.h instead. 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This code implements ARM atomics for architectures V6 and newer. 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifndef BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stdio.h> 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stdlib.h> 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/abort.h" 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/basictypes.h" // For COMPILE_ASSERT 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// The LDREXD and STREXD instructions in ARM all v7 variants or above. In v6, 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// only some variants support it. For simplicity, we only use exclusive 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 64-bit load/store in V7 or above. 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if defined(ARMV7) 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# define BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef int32_t Atomic32; 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace base { 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace subtle { 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef int64_t Atomic64; 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 32-bit low-level ops 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 old_value, 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 new_value) { 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 oldval, res; 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do { 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__( 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "ldrex %1, [%3]\n" 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "mov %0, #0\n" 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "teq %1, %4\n" 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // The following IT (if-then) instruction is needed for the subsequent 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // conditional instruction STREXEQ when compiling in THUMB mode. 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // In ARM mode, the compiler/assembler will not generate any code for it. 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "it eq\n" 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "strexeq %0, %5, [%3]\n" 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "r" (ptr), "Ir" (old_value), "r" (new_value) 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "cc"); 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } while (res); 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return oldval; 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 new_value) { 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 tmp, old; 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__( 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "1:\n" 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "ldrex %1, [%2]\n" 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "strex %0, %3, [%2]\n" 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "teq %0, #0\n" 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "bne 1b" 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "=&r" (tmp), "=&r" (old) 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "r" (ptr), "r" (new_value) 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "cc", "memory"); 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return old; 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 increment) { 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 tmp, res; 1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__( 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "1:\n" 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "ldrex %1, [%2]\n" 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "add %1, %1, %3\n" 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "strex %0, %1, [%2]\n" 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "teq %0, #0\n" 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "bne 1b" 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "=&r" (tmp), "=&r"(res) 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "r" (ptr), "r"(increment) 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "cc", "memory"); 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return res; 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void MemoryBarrier() { 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__("dmb" : : : "memory"); 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 increment) { 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 tmp, res; 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__( 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "1:\n" 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "ldrex %1, [%2]\n" 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "add %1, %1, %3\n" 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "dmb\n" 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "strex %0, %1, [%2]\n" 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "teq %0, #0\n" 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "bne 1b" 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "=&r" (tmp), "=&r"(res) 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "r" (ptr), "r"(increment) 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "cc", "memory"); 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return res; 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 old_value, 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 new_value) { 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return value; 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 old_value, 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 new_value) { 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NoBarrier_CompareAndSwap(ptr, old_value, new_value); 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ptr = value; 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ptr = value; 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ptr = value; 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return *ptr; 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 value = *ptr; 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return value; 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 Release_Load(volatile const Atomic32* ptr) { 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return *ptr; 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 64-bit versions are only available if LDREXD and STREXD instructions 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// are available. 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define BASE_HAS_ATOMIC64 1 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 old_value, 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 new_value) { 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 oldval, res; 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do { 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__( 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "ldrexd %1, [%3]\n" 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "mov %0, #0\n" 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "teq %Q1, %Q4\n" 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // The following IT (if-then) instructions are needed for the subsequent 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // conditional instructions when compiling in THUMB mode. 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // In ARM mode, the compiler/assembler will not generate any code for it. 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "it eq\n" 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "teqeq %R1, %R4\n" 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "it eq\n" 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "strexdeq %0, %5, [%3]\n" 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "=&r" (res), "=&r" (oldval), "+Q" (*ptr) 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "r" (ptr), "Ir" (old_value), "r" (new_value) 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "cc"); 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } while (res); 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return oldval; 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 new_value) { 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int store_failed; 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 old; 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__( 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "1:\n" 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "ldrexd %1, [%2]\n" 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "strexd %0, %3, [%2]\n" 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "teq %0, #0\n" 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "bne 1b" 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "=&r" (store_failed), "=&r" (old) 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "r" (ptr), "r" (new_value) 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "cc", "memory"); 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return old; 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 increment) { 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int store_failed; 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 res; 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__( 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "1:\n" 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "ldrexd %1, [%2]\n" 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "adds %Q1, %Q1, %Q3\n" 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "adc %R1, %R1, %R3\n" 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "strexd %0, %1, [%2]\n" 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "teq %0, #0\n" 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "bne 1b" 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "=&r" (store_failed), "=&r"(res) 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "r" (ptr), "r"(increment) 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "cc", "memory"); 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return res; 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, 2435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 increment) { 2445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int store_failed; 2455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 res; 2465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__( 2475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "1:\n" 2485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "ldrexd %1, [%2]\n" 2495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "adds %Q1, %Q1, %Q3\n" 2505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "adc %R1, %R1, %R3\n" 2515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "dmb\n" 2525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "strexd %0, %1, [%2]\n" 2535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "teq %0, #0\n" 2545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "bne 1b" 2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "=&r" (store_failed), "=&r"(res) 2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "r" (ptr), "r"(increment) 2575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "cc", "memory"); 2585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return res; 2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { 2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int store_failed; 2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 dummy; 2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__( 2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "1:\n" 2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Dummy load to lock cache line. 2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "ldrexd %1, [%3]\n" 2685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "strexd %0, %2, [%3]\n" 2695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "teq %0, #0\n" 2705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "bne 1b" 2715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "=&r" (store_failed), "=&r"(dummy) 2725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "r"(value), "r" (ptr) 2735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "cc", "memory"); 2745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { 2775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 res; 2785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__( 2795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "ldrexd %0, [%1]\n" 2805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "clrex\n" 2815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "=r" (res) 2825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "r"(ptr), "Q"(*ptr)); 2835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return res; 2845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD 2875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void NotImplementedFatalError(const char *function_name) { 2895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fprintf(stderr, "64-bit %s() not implemented on this platform\n", 2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) function_name); 2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tcmalloc::Abort(); 2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, 2955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 old_value, 2965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 new_value) { 2975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NotImplementedFatalError("NoBarrier_CompareAndSwap"); 2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 2995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, 3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 new_value) { 3035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NotImplementedFatalError("NoBarrier_AtomicExchange"); 3045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 3055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, 3085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 increment) { 3095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NotImplementedFatalError("NoBarrier_AtomicIncrement"); 3105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 3115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, 3145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 increment) { 3155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NotImplementedFatalError("Barrier_AtomicIncrement"); 3165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 3175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { 3205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NotImplementedFatalError("NoBarrier_Store"); 3215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { 3245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NotImplementedFatalError("NoBarrier_Load"); 3255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 3265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD 3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { 3315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NoBarrier_Store(ptr, value); 3325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 3335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { 3365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 3375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NoBarrier_Store(ptr, value); 3385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { 3415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 value = NoBarrier_Load(ptr); 3425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 3435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return value; 3445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Release_Load(volatile const Atomic64* ptr) { 3475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 3485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NoBarrier_Load(ptr); 3495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, 3525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 old_value, 3535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 new_value) { 3545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); 3555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 3565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return value; 3575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, 3605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 old_value, 3615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 new_value) { 3625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 3635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NoBarrier_CompareAndSwap(ptr, old_value, new_value); 3645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // namespace subtle ends 3675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // namespace base ends 3685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ 370