1// Copyright (c) 2011, Google Inc.
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are
6// met:
7//
8//     * Redistributions of source code must retain the above copyright
9// notice, this list of conditions and the following disclaimer.
10//     * Redistributions in binary form must reproduce the above
11// copyright notice, this list of conditions and the following disclaimer
12// in the documentation and/or other materials provided with the
13// distribution.
14//     * Neither the name of Google Inc. nor the names of its
15// contributors may be used to endorse or promote products derived from
16// this software without specific prior written permission.
17//
18// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29// ---
30//
31// Author: Sasha Levitskiy
32// based on atomicops-internals by Sanjay Ghemawat
33//
34// This file is an internal atomic implementation, use base/atomicops.h instead.
35//
36// This code implements ARM atomics for architectures V6 and  newer.
37
38#ifndef BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_
39#define BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_
40
41#include <stdio.h>
42#include <stdlib.h>
43#include "base/basictypes.h"  // For COMPILE_ASSERT
44
45// The LDREXD and STREXD instructions in ARM all v7 variants or above.  In v6,
46// only some variants support it.  For simplicity, we only use exclusive
47// 64-bit load/store in V7 or above.
48#if defined(ARMV7)
49# define BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD
50#endif
51
52typedef int32_t Atomic32;
53
54namespace base {
55namespace subtle {
56
57typedef int64_t Atomic64;
58
59// 32-bit low-level ops
60
61inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr,
62                                         Atomic32 old_value,
63                                         Atomic32 new_value) {
64  Atomic32 oldval, res;
65  do {
66    __asm__ __volatile__(
67    "ldrex   %1, [%3]\n"
68    "mov     %0, #0\n"
69    "teq     %1, %4\n"
70    // The following IT (if-then) instruction is needed for the subsequent
71    // conditional instruction STREXEQ when compiling in THUMB mode.
72    // In ARM mode, the compiler/assembler will not generate any code for it.
73    "it      eq\n"
74    "strexeq %0, %5, [%3]\n"
75        : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr)
76        : "r" (ptr), "Ir" (old_value), "r" (new_value)
77        : "cc");
78  } while (res);
79  return oldval;
80}
81
82inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr,
83                                         Atomic32 new_value) {
84  Atomic32 tmp, old;
85  __asm__ __volatile__(
86      "1:\n"
87      "ldrex  %1, [%2]\n"
88      "strex  %0, %3, [%2]\n"
89      "teq    %0, #0\n"
90      "bne    1b"
91      : "=&r" (tmp), "=&r" (old)
92      : "r" (ptr), "r" (new_value)
93      : "cc", "memory");
94  return old;
95}
96
97inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr,
98                                          Atomic32 increment) {
99  Atomic32 tmp, res;
100  __asm__ __volatile__(
101      "1:\n"
102      "ldrex  %1, [%2]\n"
103      "add    %1, %1, %3\n"
104      "strex  %0, %1, [%2]\n"
105      "teq    %0, #0\n"
106      "bne    1b"
107      : "=&r" (tmp), "=&r"(res)
108      : "r" (ptr), "r"(increment)
109      : "cc", "memory");
110  return res;
111}
112
113inline void MemoryBarrier() {
114  __asm__ __volatile__("dmb" : : : "memory");
115}
116
117inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr,
118                                        Atomic32 increment) {
119  Atomic32 tmp, res;
120  __asm__ __volatile__(
121      "1:\n"
122      "ldrex  %1, [%2]\n"
123      "add    %1, %1, %3\n"
124      "dmb\n"
125      "strex  %0, %1, [%2]\n"
126      "teq    %0, #0\n"
127      "bne    1b"
128      : "=&r" (tmp), "=&r"(res)
129      : "r" (ptr), "r"(increment)
130      : "cc", "memory");
131  return res;
132}
133
134inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr,
135                                       Atomic32 old_value,
136                                       Atomic32 new_value) {
137  Atomic32 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value);
138  MemoryBarrier();
139  return value;
140}
141
142inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr,
143                                       Atomic32 old_value,
144                                       Atomic32 new_value) {
145  MemoryBarrier();
146  return NoBarrier_CompareAndSwap(ptr, old_value, new_value);
147}
148
149inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) {
150  *ptr = value;
151}
152
153inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) {
154  *ptr = value;
155  MemoryBarrier();
156}
157
158inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) {
159  MemoryBarrier();
160  *ptr = value;
161}
162
163inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) {
164  return *ptr;
165}
166
167inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) {
168  Atomic32 value = *ptr;
169  MemoryBarrier();
170  return value;
171}
172
173inline Atomic32 Release_Load(volatile const Atomic32* ptr) {
174  MemoryBarrier();
175  return *ptr;
176}
177
178// 64-bit versions are only available if LDREXD and STREXD instructions
179// are available.
180#ifdef BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD
181
182#define BASE_HAS_ATOMIC64 1
183
184inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr,
185                                         Atomic64 old_value,
186                                         Atomic64 new_value) {
187  Atomic64 oldval, res;
188  do {
189    __asm__ __volatile__(
190    "ldrexd   %1, [%3]\n"
191    "mov      %0, #0\n"
192    "teq      %Q1, %Q4\n"
193    // The following IT (if-then) instructions are needed for the subsequent
194    // conditional instructions when compiling in THUMB mode.
195    // In ARM mode, the compiler/assembler will not generate any code for it.
196    "it       eq\n"
197    "teqeq    %R1, %R4\n"
198    "it       eq\n"
199    "strexdeq %0, %5, [%3]\n"
200        : "=&r" (res), "=&r" (oldval), "+Q" (*ptr)
201        : "r" (ptr), "Ir" (old_value), "r" (new_value)
202        : "cc");
203  } while (res);
204  return oldval;
205}
206
207inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr,
208                                         Atomic64 new_value) {
209  int store_failed;
210  Atomic64 old;
211  __asm__ __volatile__(
212      "1:\n"
213      "ldrexd  %1, [%2]\n"
214      "strexd  %0, %3, [%2]\n"
215      "teq     %0, #0\n"
216      "bne     1b"
217      : "=&r" (store_failed), "=&r" (old)
218      : "r" (ptr), "r" (new_value)
219      : "cc", "memory");
220  return old;
221}
222
223inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr,
224                                          Atomic64 increment) {
225  int store_failed;
226  Atomic64 res;
227  __asm__ __volatile__(
228      "1:\n"
229      "ldrexd  %1, [%2]\n"
230      "adds    %Q1, %Q1, %Q3\n"
231      "adc     %R1, %R1, %R3\n"
232      "strexd  %0, %1, [%2]\n"
233      "teq     %0, #0\n"
234      "bne     1b"
235      : "=&r" (store_failed), "=&r"(res)
236      : "r" (ptr), "r"(increment)
237      : "cc", "memory");
238  return res;
239}
240
241inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr,
242                                        Atomic64 increment) {
243  int store_failed;
244  Atomic64 res;
245  __asm__ __volatile__(
246      "1:\n"
247      "ldrexd  %1, [%2]\n"
248      "adds    %Q1, %Q1, %Q3\n"
249      "adc     %R1, %R1, %R3\n"
250      "dmb\n"
251      "strexd  %0, %1, [%2]\n"
252      "teq     %0, #0\n"
253      "bne     1b"
254      : "=&r" (store_failed), "=&r"(res)
255      : "r" (ptr), "r"(increment)
256      : "cc", "memory");
257  return res;
258}
259
260inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) {
261  int store_failed;
262  Atomic64 dummy;
263  __asm__ __volatile__(
264      "1:\n"
265      // Dummy load to lock cache line.
266      "ldrexd  %1, [%3]\n"
267      "strexd  %0, %2, [%3]\n"
268      "teq     %0, #0\n"
269      "bne     1b"
270      : "=&r" (store_failed), "=&r"(dummy)
271      : "r"(value), "r" (ptr)
272      : "cc", "memory");
273}
274
275inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) {
276  Atomic64 res;
277  __asm__ __volatile__(
278  "ldrexd   %0, [%1]\n"
279  "clrex\n"
280      : "=r" (res)
281      : "r"(ptr), "Q"(*ptr));
282  return res;
283}
284
285#else // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD
286
287inline void NotImplementedFatalError(const char *function_name) {
288  fprintf(stderr, "64-bit %s() not implemented on this platform\n",
289          function_name);
290  abort();
291}
292
293inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr,
294                                         Atomic64 old_value,
295                                         Atomic64 new_value) {
296  NotImplementedFatalError("NoBarrier_CompareAndSwap");
297  return 0;
298}
299
300inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr,
301                                         Atomic64 new_value) {
302  NotImplementedFatalError("NoBarrier_AtomicExchange");
303  return 0;
304}
305
306inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr,
307                                          Atomic64 increment) {
308  NotImplementedFatalError("NoBarrier_AtomicIncrement");
309  return 0;
310}
311
312inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr,
313                                        Atomic64 increment) {
314  NotImplementedFatalError("Barrier_AtomicIncrement");
315  return 0;
316}
317
318inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) {
319  NotImplementedFatalError("NoBarrier_Store");
320}
321
322inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) {
323  NotImplementedFatalError("NoBarrier_Load");
324  return 0;
325}
326
327#endif // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD
328
329inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) {
330  NoBarrier_Store(ptr, value);
331  MemoryBarrier();
332}
333
334inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) {
335  MemoryBarrier();
336  NoBarrier_Store(ptr, value);
337}
338
339inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) {
340  Atomic64 value = NoBarrier_Load(ptr);
341  MemoryBarrier();
342  return value;
343}
344
345inline Atomic64 Release_Load(volatile const Atomic64* ptr) {
346  MemoryBarrier();
347  return NoBarrier_Load(ptr);
348}
349
350inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr,
351                                       Atomic64 old_value,
352                                       Atomic64 new_value) {
353  Atomic64 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value);
354  MemoryBarrier();
355  return value;
356}
357
358inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr,
359                                       Atomic64 old_value,
360                                       Atomic64 new_value) {
361  MemoryBarrier();
362  return NoBarrier_CompareAndSwap(ptr, old_value, new_value);
363}
364
365}  // namespace subtle ends
366}  // namespace base ends
367
368#endif  // BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_
369