u_atomic.h revision 10b07665be5cff9fa9f03b0f7db459f3b380570d 
1/**
2 * Many similar implementations exist. See for example libwsbm
3 * or the linux kernel include/atomic.h
4 *
5 * No copyright claimed on this file.
6 *
7 */
8
9#ifndef U_ATOMIC_H
10#define U_ATOMIC_H
11
12#include "pipe/p_compiler.h"
13#include "pipe/p_defines.h"
14
15/* Favor OS-provided implementations.
16 *
17 * Where no OS-provided implementation is available, fall back to
18 * locally coded assembly, compiler intrinsic or ultimately a
19 * mutex-based implementation.
20 */
21#if defined(PIPE_OS_SOLARIS)
22#define PIPE_ATOMIC_OS_SOLARIS
23#elif defined(PIPE_CC_MSVC)
24#define PIPE_ATOMIC_MSVC_INTRINSIC
25#elif (defined(PIPE_CC_MSVC) && defined(PIPE_ARCH_X86))
26#define PIPE_ATOMIC_ASM_MSVC_X86
27#elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86))
28#define PIPE_ATOMIC_ASM_GCC_X86
29#elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64))
30#define PIPE_ATOMIC_ASM_GCC_X86_64
31#elif defined(PIPE_CC_GCC) && (PIPE_CC_GCC_VERSION >= 401)
32#define PIPE_ATOMIC_GCC_INTRINSIC
33#else
34#error "Unsupported platform"
35#endif
36
37
38#if defined(PIPE_ATOMIC_ASM_GCC_X86_64)
39#define PIPE_ATOMIC "GCC x86_64 assembly"
40
41#ifdef __cplusplus
42extern "C" {
43#endif
44
45#define p_atomic_set(_v, _i) (*(_v) = (_i))
46#define p_atomic_read(_v) (*(_v))
47
48static INLINE boolean
49p_atomic_dec_zero(int32_t *v)
50{
51   unsigned char c;
52
53   __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
54			::"memory");
55
56   return c != 0;
57}
58
59static INLINE void
60p_atomic_inc(int32_t *v)
61{
62   __asm__ __volatile__("lock; incl %0":"+m"(*v));
63}
64
65static INLINE void
66p_atomic_dec(int32_t *v)
67{
68   __asm__ __volatile__("lock; decl %0":"+m"(*v));
69}
70
71static INLINE int32_t
72p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
73{
74   return __sync_val_compare_and_swap(v, old, _new);
75}
76
77#ifdef __cplusplus
78}
79#endif
80
81#endif /* PIPE_ATOMIC_ASM_GCC_X86_64 */
82
83
84#if defined(PIPE_ATOMIC_ASM_GCC_X86)
85
86#define PIPE_ATOMIC "GCC x86 assembly"
87
88#ifdef __cplusplus
89extern "C" {
90#endif
91
92#define p_atomic_set(_v, _i) (*(_v) = (_i))
93#define p_atomic_read(_v) (*(_v))
94
95static INLINE boolean
96p_atomic_dec_zero(int32_t *v)
97{
98   unsigned char c;
99
100   __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
101			::"memory");
102
103   return c != 0;
104}
105
106static INLINE void
107p_atomic_inc(int32_t *v)
108{
109   __asm__ __volatile__("lock; incl %0":"+m"(*v));
110}
111
112static INLINE void
113p_atomic_dec(int32_t *v)
114{
115   __asm__ __volatile__("lock; decl %0":"+m"(*v));
116}
117
118static INLINE int32_t
119p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
120{
121   return __sync_val_compare_and_swap(v, old, _new);
122}
123
124#ifdef __cplusplus
125}
126#endif
127
128#endif
129
130
131
132/* Implementation using GCC-provided synchronization intrinsics
133 */
134#if defined(PIPE_ATOMIC_GCC_INTRINSIC)
135
136#define PIPE_ATOMIC "GCC Sync Intrinsics"
137
138#ifdef __cplusplus
139extern "C" {
140#endif
141
142#define p_atomic_set(_v, _i) (*(_v) = (_i))
143#define p_atomic_read(_v) (*(_v))
144
145static INLINE boolean
146p_atomic_dec_zero(int32_t *v)
147{
148   return (__sync_sub_and_fetch(v, 1) == 0);
149}
150
151static INLINE void
152p_atomic_inc(int32_t *v)
153{
154   (void) __sync_add_and_fetch(v, 1);
155}
156
157static INLINE void
158p_atomic_dec(int32_t *v)
159{
160   (void) __sync_sub_and_fetch(v, 1);
161}
162
163static INLINE int32_t
164p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
165{
166   return __sync_val_compare_and_swap(v, old, _new);
167}
168
169#ifdef __cplusplus
170}
171#endif
172
173#endif
174
175
176
177/* Unlocked version for single threaded environments, such as some
178 * windows kernel modules.
179 */
180#if defined(PIPE_ATOMIC_OS_UNLOCKED)
181
182#define PIPE_ATOMIC "Unlocked"
183
184#define p_atomic_set(_v, _i) (*(_v) = (_i))
185#define p_atomic_read(_v) (*(_v))
186#define p_atomic_dec_zero(_v) ((boolean) --(*(_v)))
187#define p_atomic_inc(_v) ((void) (*(_v))++)
188#define p_atomic_dec(_v) ((void) (*(_v))--)
189#define p_atomic_cmpxchg(_v, old, _new) (*(_v) == old ? *(_v) = (_new) : *(_v))
190
191#endif
192
193
194/* Locally coded assembly for MSVC on x86:
195 */
196#if defined(PIPE_ATOMIC_ASM_MSVC_X86)
197
198#define PIPE_ATOMIC "MSVC x86 assembly"
199
200#ifdef __cplusplus
201extern "C" {
202#endif
203
204#define p_atomic_set(_v, _i) (*(_v) = (_i))
205#define p_atomic_read(_v) (*(_v))
206
207static INLINE boolean
208p_atomic_dec_zero(int32_t *v)
209{
210   unsigned char c;
211
212   __asm {
213      mov       eax, [v]
214      lock dec  dword ptr [eax]
215      sete      byte ptr [c]
216   }
217
218   return c != 0;
219}
220
221static INLINE void
222p_atomic_inc(int32_t *v)
223{
224   __asm {
225      mov       eax, [v]
226      lock inc  dword ptr [eax]
227   }
228}
229
230static INLINE void
231p_atomic_dec(int32_t *v)
232{
233   __asm {
234      mov       eax, [v]
235      lock dec  dword ptr [eax]
236   }
237}
238
239static INLINE int32_t
240p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
241{
242   int32_t orig;
243
244   __asm {
245      mov ecx, [v]
246      mov eax, [old]
247      mov edx, [_new]
248      lock cmpxchg [ecx], edx
249      mov [orig], eax
250   }
251
252   return orig;
253}
254
255#ifdef __cplusplus
256}
257#endif
258
259#endif
260
261
262#if defined(PIPE_ATOMIC_MSVC_INTRINSIC)
263
264#define PIPE_ATOMIC "MSVC Intrinsics"
265
266#include <intrin.h>
267
268#pragma intrinsic(_InterlockedIncrement)
269#pragma intrinsic(_InterlockedDecrement)
270#pragma intrinsic(_InterlockedCompareExchange)
271
272#ifdef __cplusplus
273extern "C" {
274#endif
275
276#define p_atomic_set(_v, _i) (*(_v) = (_i))
277#define p_atomic_read(_v) (*(_v))
278
279static INLINE boolean
280p_atomic_dec_zero(int32_t *v)
281{
282   return _InterlockedDecrement((long *)v) == 0;
283}
284
285static INLINE void
286p_atomic_inc(int32_t *v)
287{
288   _InterlockedIncrement((long *)v);
289}
290
291static INLINE void
292p_atomic_dec(int32_t *v)
293{
294   _InterlockedDecrement((long *)v);
295}
296
297static INLINE int32_t
298p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
299{
300   return _InterlockedCompareExchange((long *)v, _new, old);
301}
302
303#ifdef __cplusplus
304}
305#endif
306
307#endif
308
309#if defined(PIPE_ATOMIC_OS_SOLARIS)
310
311#define PIPE_ATOMIC "Solaris OS atomic functions"
312
313#include <atomic.h>
314
315#ifdef __cplusplus
316extern "C" {
317#endif
318
319#define p_atomic_set(_v, _i) (*(_v) = (_i))
320#define p_atomic_read(_v) (*(_v))
321
322static INLINE boolean
323p_atomic_dec_zero(int32_t *v)
324{
325   uint32_t n = atomic_dec_32_nv((uint32_t *) v);
326
327   return n != 0;
328}
329
330#define p_atomic_inc(_v) atomic_inc_32((uint32_t *) _v)
331#define p_atomic_dec(_v) atomic_dec_32((uint32_t *) _v)
332
333#define p_atomic_cmpxchg(_v, _old, _new) \
334	atomic_cas_32( (uint32_t *) _v, (uint32_t) _old, (uint32_t) _new)
335
336#ifdef __cplusplus
337}
338#endif
339
340#endif
341
342
343#ifndef PIPE_ATOMIC
344#error "No pipe_atomic implementation selected"
345#endif
346
347
348
349#endif /* U_ATOMIC_H */
350