bionic_atomic_arm.h revision e91f71783bf86cb10ad52f5618ad07d4e7152b2f 
1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16#ifndef BIONIC_ATOMIC_ARM_H
17#define BIONIC_ATOMIC_ARM_H
18
19#include <machine/cpu-features.h>
20
21/* Some of the harware instructions used below are not available in Thumb-1
22 * mode (they are if you build in ARM or Thumb-2 mode though). To solve this
23 * problem, we're going to use the same technique than libatomics_ops,
24 * which is to temporarily switch to ARM, do the operation, then switch
25 * back to Thumb-1.
26 *
27 * This results in two 'bx' jumps, just like a normal function call, but
28 * everything is kept inlined, avoids loading or computing the function's
29 * address, and prevents a little I-cache trashing too.
30 *
31 * However, it is highly recommended to avoid compiling any C library source
32 * file that use these functions in Thumb-1 mode.
33 *
34 * Define three helper macros to implement this:
35 */
36#if defined(__thumb__) && !defined(__thumb2__)
37#  define  __ATOMIC_SWITCH_TO_ARM \
38            "adr r3, 5f\n" \
39            "bx  r3\n" \
40            ".align\n" \
41            ".arm\n" \
42        "5:\n"
43/* note: the leading \n below is intentional */
44#  define __ATOMIC_SWITCH_TO_THUMB \
45            "\n" \
46            "adr r3, 6f\n" \
47            "bx  r3\n" \
48            ".thumb" \
49        "6:\n"
50
51#  define __ATOMIC_CLOBBERS   "r3"  /* list of clobbered registers */
52
53/* Warn the user that ARM mode should really be preferred! */
54#  warning Rebuilding this source file in ARM mode is highly recommended for performance!!
55
56#else
57#  define  __ATOMIC_SWITCH_TO_ARM   /* nothing */
58#  define  __ATOMIC_SWITCH_TO_THUMB /* nothing */
59#  define  __ATOMIC_CLOBBERS        /* nothing */
60#endif
61
62
63/* Define a full memory barrier, this is only needed if we build the
64 * platform for a multi-core device. For the record, using a 'dmb'
65 * instruction on a Nexus One device can take up to 180 ns even if
66 * it is completely un-necessary on this device.
67 *
68 * NOTE: This is where the platform and NDK headers atomic headers are
69 *        going to diverge. With the NDK, we don't know if the generated
70 *        code is going to run on a single or multi-core device, so we
71 *        need to be cautious.
72 *
73 *        I.e. on single-core devices, the helper immediately returns,
74 *        on multi-core devices, it uses "dmb" or any other means to
75 *        perform a full-memory barrier.
76 *
77 * There are three cases to consider for the platform:
78 *
79 *    - multi-core ARMv7-A       => use the 'dmb' hardware instruction
80 *    - multi-core ARMv6         => use the coprocessor
81 *    - single core ARMv6+       => do not use any hardware barrier
82 */
83#if defined(ANDROID_SMP) && ANDROID_SMP == 1
84
85/* Sanity check, multi-core is only supported starting from ARMv6 */
86#  if __ARM_ARCH__ < 6
87#    error ANDROID_SMP should not be set to 1 for an ARM architecture less than 6
88#  endif
89
90#  ifdef __ARM_HAVE_DMB
91/* For ARMv7-A, we can use the 'dmb' instruction directly */
92__ATOMIC_INLINE__ void
93__bionic_memory_barrier(void)
94{
95    /* Note: we always build in ARM or Thumb-2 on ARMv7-A, so don't
96     * bother with __ATOMIC_SWITCH_TO_ARM */
97    __asm__ __volatile__ ( "dmb" : : : "memory" );
98}
99#  else /* !__ARM_HAVE_DMB */
100/* Otherwise, i.e. for multi-core ARMv6, we need to use the coprocessor,
101 * which requires the use of a general-purpose register, which is slightly
102 * less efficient.
103 */
104__ATOMIC_INLINE__ void
105__bionic_memory_barrier(void)
106{
107    __asm__ __volatile__ (
108        __SWITCH_TO_ARM
109        "mcr p15, 0, %0, c7, c10, 5"
110        __SWITCH_TO_THUMB
111        : : "r" (0) : __ATOMIC_CLOBBERS "memory");
112}
113#  endif /* !__ARM_HAVE_DMB */
114#else /* !ANDROID_SMP */
115__ATOMIC_INLINE__ void
116__bionic_memory_barrier(void)
117{
118    /* A simple compiler barrier */
119    __asm__ __volatile__ ( "" : : : "memory" );
120}
121#endif /* !ANDROID_SMP */
122
123#ifndef __ARM_HAVE_LDREX_STREX
124#error Only ARM devices which have LDREX / STREX are supported
125#endif
126
127/* Compare-and-swap, without any explicit barriers. Note that this functions
128 * returns 0 on success, and 1 on failure. The opposite convention is typically
129 * used on other platforms.
130 */
131__ATOMIC_INLINE__ int
132__bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr)
133{
134    int32_t prev, status;
135    do {
136        __asm__ __volatile__ (
137            __ATOMIC_SWITCH_TO_ARM
138            "ldrex %0, [%3]\n"
139            "mov %1, #0\n"
140            "teq %0, %4\n"
141#ifdef __thumb2__
142            "it eq\n"
143#endif
144            "strexeq %1, %5, [%3]"
145            __ATOMIC_SWITCH_TO_THUMB
146            : "=&r" (prev), "=&r" (status), "+m"(*ptr)
147            : "r" (ptr), "Ir" (old_value), "r" (new_value)
148            : __ATOMIC_CLOBBERS "cc");
149    } while (__builtin_expect(status != 0, 0));
150    return prev != old_value;
151}
152
153/* Swap operation, without any explicit barriers. */
154__ATOMIC_INLINE__ int32_t
155__bionic_swap(int32_t new_value, volatile int32_t* ptr)
156{
157    int32_t prev, status;
158    do {
159        __asm__ __volatile__ (
160            __ATOMIC_SWITCH_TO_ARM
161            "ldrex %0, [%3]\n"
162            "strex %1, %4, [%3]"
163            __ATOMIC_SWITCH_TO_THUMB
164            : "=&r" (prev), "=&r" (status), "+m" (*ptr)
165            : "r" (ptr), "r" (new_value)
166            : __ATOMIC_CLOBBERS "cc");
167    } while (__builtin_expect(status != 0, 0));
168    return prev;
169}
170
171/* Atomic increment - without any barriers
172 * This returns the old value
173 */
174__ATOMIC_INLINE__ int32_t
175__bionic_atomic_inc(volatile int32_t* ptr)
176{
177    int32_t prev, tmp, status;
178    do {
179        __asm__ __volatile__ (
180            __ATOMIC_SWITCH_TO_ARM
181            "ldrex %0, [%4]\n"
182            "add %1, %0, #1\n"
183            "strex %2, %1, [%4]"
184            __ATOMIC_SWITCH_TO_THUMB
185            : "=&r" (prev), "=&r" (tmp), "=&r" (status), "+m"(*ptr)
186            : "r" (ptr)
187            : __ATOMIC_CLOBBERS "cc");
188    } while (__builtin_expect(status != 0, 0));
189    return prev;
190}
191
192/* Atomic decrement - without any barriers
193 * This returns the old value.
194 */
195__ATOMIC_INLINE__ int32_t
196__bionic_atomic_dec(volatile int32_t* ptr)
197{
198    int32_t prev, tmp, status;
199    do {
200        __asm__ __volatile__ (
201            __ATOMIC_SWITCH_TO_ARM
202            "ldrex %0, [%4]\n"
203            "sub %1, %0, #1\n"
204            "strex %2, %1, [%4]"
205            __ATOMIC_SWITCH_TO_THUMB
206            : "=&r" (prev), "=&r" (tmp), "=&r" (status), "+m"(*ptr)
207            : "r" (ptr)
208            : __ATOMIC_CLOBBERS "cc");
209    } while (__builtin_expect(status != 0, 0));
210    return prev;
211}
212
213#endif /* SYS_ATOMICS_ARM_H */
214