atomic.h revision 7427525c28d58c423a68930160e3b0fe577fe953 
1/******************************************************************************/
2#ifdef JEMALLOC_H_TYPES
3
4#endif /* JEMALLOC_H_TYPES */
5/******************************************************************************/
6#ifdef JEMALLOC_H_STRUCTS
7
8#endif /* JEMALLOC_H_STRUCTS */
9/******************************************************************************/
10#ifdef JEMALLOC_H_EXTERNS
11
12#define	atomic_read_uint64(p)	atomic_add_uint64(p, 0)
13#define	atomic_read_uint32(p)	atomic_add_uint32(p, 0)
14
15#if (LG_SIZEOF_PTR == 3)
16#  define atomic_read_z(p)						\
17    (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)0)
18#  define atomic_add_z(p, x)						\
19    (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x)
20#  define atomic_sub_z(p, x)						\
21    (size_t)atomic_sub_uint64((uint64_t *)p, (uint64_t)x)
22#elif (LG_SIZEOF_PTR == 2)
23#  define atomic_read_z(p)						\
24    (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)0)
25#  define atomic_add_z(p, x)						\
26    (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x)
27#  define atomic_sub_z(p, x)						\
28    (size_t)atomic_sub_uint32((uint32_t *)p, (uint32_t)x)
29#endif
30
31#endif /* JEMALLOC_H_EXTERNS */
32/******************************************************************************/
33#ifdef JEMALLOC_H_INLINES
34
35#ifndef JEMALLOC_ENABLE_INLINE
36uint64_t	atomic_add_uint64(uint64_t *p, uint64_t x);
37uint64_t	atomic_sub_uint64(uint64_t *p, uint64_t x);
38uint32_t	atomic_add_uint32(uint32_t *p, uint32_t x);
39uint32_t	atomic_sub_uint32(uint32_t *p, uint32_t x);
40#endif
41
42#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_))
43/******************************************************************************/
44/* 64-bit operations. */
45#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
46JEMALLOC_INLINE uint64_t
47atomic_add_uint64(uint64_t *p, uint64_t x)
48{
49
50	return (__sync_add_and_fetch(p, x));
51}
52
53JEMALLOC_INLINE uint64_t
54atomic_sub_uint64(uint64_t *p, uint64_t x)
55{
56
57	return (__sync_sub_and_fetch(p, x));
58}
59#elif (defined(JEMALLOC_OSATOMIC))
60JEMALLOC_INLINE uint64_t
61atomic_add_uint64(uint64_t *p, uint64_t x)
62{
63
64	return (OSAtomicAdd64((int64_t)x, (int64_t *)p));
65}
66
67JEMALLOC_INLINE uint64_t
68atomic_sub_uint64(uint64_t *p, uint64_t x)
69{
70
71	return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
72}
73#elif (defined(__amd64_) || defined(__x86_64__))
74JEMALLOC_INLINE uint64_t
75atomic_add_uint64(uint64_t *p, uint64_t x)
76{
77
78	asm volatile (
79	    "lock; xaddq %0, %1;"
80	    : "+r" (x), "=m" (*p) /* Outputs. */
81	    : "m" (*p) /* Inputs. */
82	    );
83
84	return (x);
85}
86
87JEMALLOC_INLINE uint64_t
88atomic_sub_uint64(uint64_t *p, uint64_t x)
89{
90
91	x = (uint64_t)(-(int64_t)x);
92	asm volatile (
93	    "lock; xaddq %0, %1;"
94	    : "+r" (x), "=m" (*p) /* Outputs. */
95	    : "m" (*p) /* Inputs. */
96	    );
97
98	return (x);
99}
100#else
101#  if (LG_SIZEOF_PTR == 3)
102#    error "Missing implementation for 64-bit atomic operations"
103#  endif
104#endif
105
106/******************************************************************************/
107/* 32-bit operations. */
108#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4
109JEMALLOC_INLINE uint32_t
110atomic_add_uint32(uint32_t *p, uint32_t x)
111{
112
113	return (__sync_add_and_fetch(p, x));
114}
115
116JEMALLOC_INLINE uint32_t
117atomic_sub_uint32(uint32_t *p, uint32_t x)
118{
119
120	return (__sync_sub_and_fetch(p, x));
121}
122#elif (defined(JEMALLOC_OSATOMIC))
123JEMALLOC_INLINE uint32_t
124atomic_add_uint32(uint32_t *p, uint32_t x)
125{
126
127	return (OSAtomicAdd32((int32_t)x, (int32_t *)p));
128}
129
130JEMALLOC_INLINE uint32_t
131atomic_sub_uint32(uint32_t *p, uint32_t x)
132{
133
134	return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p));
135}
136#elif (defined(__i386__) || defined(__amd64_) || defined(__x86_64__))
137JEMALLOC_INLINE uint32_t
138atomic_add_uint32(uint32_t *p, uint32_t x)
139{
140
141	asm volatile (
142	    "lock; xaddl %0, %1;"
143	    : "+r" (x), "=m" (*p) /* Outputs. */
144	    : "m" (*p) /* Inputs. */
145	    );
146
147	return (x);
148}
149
150JEMALLOC_INLINE uint32_t
151atomic_sub_uint32(uint32_t *p, uint32_t x)
152{
153
154	x = (uint32_t)(-(int32_t)x);
155	asm volatile (
156	    "lock; xaddl %0, %1;"
157	    : "+r" (x), "=m" (*p) /* Outputs. */
158	    : "m" (*p) /* Inputs. */
159	    );
160
161	return (x);
162}
163#else
164#  error "Missing implementation for 32-bit atomic operations"
165#endif
166#endif
167
168#endif /* JEMALLOC_H_INLINES */
169/******************************************************************************/
170