atomic.h revision b57d3ec571c6551231be62b7bf92c084a8c8291c 
1/******************************************************************************/
2#ifdef JEMALLOC_H_TYPES
3
4#endif /* JEMALLOC_H_TYPES */
5/******************************************************************************/
6#ifdef JEMALLOC_H_STRUCTS
7
8#endif /* JEMALLOC_H_STRUCTS */
9/******************************************************************************/
10#ifdef JEMALLOC_H_EXTERNS
11
12#define	atomic_read_uint64(p)	atomic_add_uint64(p, 0)
13#define	atomic_read_uint32(p)	atomic_add_uint32(p, 0)
14#define	atomic_read_z(p)	atomic_add_z(p, 0)
15#define	atomic_read_u(p)	atomic_add_u(p, 0)
16
17#endif /* JEMALLOC_H_EXTERNS */
18/******************************************************************************/
19#ifdef JEMALLOC_H_INLINES
20
21#ifndef JEMALLOC_ENABLE_INLINE
22uint64_t	atomic_add_uint64(uint64_t *p, uint64_t x);
23uint64_t	atomic_sub_uint64(uint64_t *p, uint64_t x);
24uint32_t	atomic_add_uint32(uint32_t *p, uint32_t x);
25uint32_t	atomic_sub_uint32(uint32_t *p, uint32_t x);
26size_t	atomic_add_z(size_t *p, size_t x);
27size_t	atomic_sub_z(size_t *p, size_t x);
28unsigned	atomic_add_u(unsigned *p, unsigned x);
29unsigned	atomic_sub_u(unsigned *p, unsigned x);
30#endif
31
32#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_))
33/******************************************************************************/
34/* 64-bit operations. */
35#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
36#  ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
37JEMALLOC_INLINE uint64_t
38atomic_add_uint64(uint64_t *p, uint64_t x)
39{
40
41	return (__sync_add_and_fetch(p, x));
42}
43
44JEMALLOC_INLINE uint64_t
45atomic_sub_uint64(uint64_t *p, uint64_t x)
46{
47
48	return (__sync_sub_and_fetch(p, x));
49}
50#elif (defined(JEMALLOC_OSATOMIC))
51JEMALLOC_INLINE uint64_t
52atomic_add_uint64(uint64_t *p, uint64_t x)
53{
54
55	return (OSAtomicAdd64((int64_t)x, (int64_t *)p));
56}
57
58JEMALLOC_INLINE uint64_t
59atomic_sub_uint64(uint64_t *p, uint64_t x)
60{
61
62	return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
63}
64#  elif (defined(__amd64__) || defined(__x86_64__))
65JEMALLOC_INLINE uint64_t
66atomic_add_uint64(uint64_t *p, uint64_t x)
67{
68
69	asm volatile (
70	    "lock; xaddq %0, %1;"
71	    : "+r" (x), "=m" (*p) /* Outputs. */
72	    : "m" (*p) /* Inputs. */
73	    );
74
75	return (x);
76}
77
78JEMALLOC_INLINE uint64_t
79atomic_sub_uint64(uint64_t *p, uint64_t x)
80{
81
82	x = (uint64_t)(-(int64_t)x);
83	asm volatile (
84	    "lock; xaddq %0, %1;"
85	    : "+r" (x), "=m" (*p) /* Outputs. */
86	    : "m" (*p) /* Inputs. */
87	    );
88
89	return (x);
90}
91#  elif (defined(JEMALLOC_ATOMIC9))
92JEMALLOC_INLINE uint64_t
93atomic_add_uint64(uint64_t *p, uint64_t x)
94{
95
96	/*
97	 * atomic_fetchadd_64() doesn't exist, but we only ever use this
98	 * function on LP64 systems, so atomic_fetchadd_long() will do.
99	 */
100	assert(sizeof(uint64_t) == sizeof(unsigned long));
101
102	return (atomic_fetchadd_long(p, (unsigned long)x) + x);
103}
104
105JEMALLOC_INLINE uint64_t
106atomic_sub_uint64(uint64_t *p, uint64_t x)
107{
108
109	assert(sizeof(uint64_t) == sizeof(unsigned long));
110
111	return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x);
112}
113#  elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
114JEMALLOC_INLINE uint64_t
115atomic_add_uint64(uint64_t *p, uint64_t x)
116{
117
118	return (__sync_add_and_fetch(p, x));
119}
120
121JEMALLOC_INLINE uint64_t
122atomic_sub_uint64(uint64_t *p, uint64_t x)
123{
124
125	return (__sync_sub_and_fetch(p, x));
126}
127#  else
128#    error "Missing implementation for 64-bit atomic operations"
129#  endif
130#endif
131
132/******************************************************************************/
133/* 32-bit operations. */
134#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4
135JEMALLOC_INLINE uint32_t
136atomic_add_uint32(uint32_t *p, uint32_t x)
137{
138
139	return (__sync_add_and_fetch(p, x));
140}
141
142JEMALLOC_INLINE uint32_t
143atomic_sub_uint32(uint32_t *p, uint32_t x)
144{
145
146	return (__sync_sub_and_fetch(p, x));
147}
148#elif (defined(JEMALLOC_OSATOMIC))
149JEMALLOC_INLINE uint32_t
150atomic_add_uint32(uint32_t *p, uint32_t x)
151{
152
153	return (OSAtomicAdd32((int32_t)x, (int32_t *)p));
154}
155
156JEMALLOC_INLINE uint32_t
157atomic_sub_uint32(uint32_t *p, uint32_t x)
158{
159
160	return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p));
161}
162#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
163JEMALLOC_INLINE uint32_t
164atomic_add_uint32(uint32_t *p, uint32_t x)
165{
166
167	asm volatile (
168	    "lock; xaddl %0, %1;"
169	    : "+r" (x), "=m" (*p) /* Outputs. */
170	    : "m" (*p) /* Inputs. */
171	    );
172
173	return (x);
174}
175
176JEMALLOC_INLINE uint32_t
177atomic_sub_uint32(uint32_t *p, uint32_t x)
178{
179
180	x = (uint32_t)(-(int32_t)x);
181	asm volatile (
182	    "lock; xaddl %0, %1;"
183	    : "+r" (x), "=m" (*p) /* Outputs. */
184	    : "m" (*p) /* Inputs. */
185	    );
186
187	return (x);
188}
189#elif (defined(JEMALLOC_ATOMIC9))
190JEMALLOC_INLINE uint32_t
191atomic_add_uint32(uint32_t *p, uint32_t x)
192{
193
194	return (atomic_fetchadd_32(p, x) + x);
195}
196
197JEMALLOC_INLINE uint32_t
198atomic_sub_uint32(uint32_t *p, uint32_t x)
199{
200
201	return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x);
202}
203#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
204JEMALLOC_INLINE uint32_t
205atomic_add_uint32(uint32_t *p, uint32_t x)
206{
207
208	return (__sync_add_and_fetch(p, x));
209}
210
211JEMALLOC_INLINE uint32_t
212atomic_sub_uint32(uint32_t *p, uint32_t x)
213{
214
215	return (__sync_sub_and_fetch(p, x));
216}
217#else
218#  error "Missing implementation for 32-bit atomic operations"
219#endif
220
221/******************************************************************************/
222/* size_t operations. */
223JEMALLOC_INLINE size_t
224atomic_add_z(size_t *p, size_t x)
225{
226
227#if (LG_SIZEOF_PTR == 3)
228	return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
229#elif (LG_SIZEOF_PTR == 2)
230	return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
231#endif
232}
233
234JEMALLOC_INLINE size_t
235atomic_sub_z(size_t *p, size_t x)
236{
237
238#if (LG_SIZEOF_PTR == 3)
239	return ((size_t)atomic_add_uint64((uint64_t *)p,
240	    (uint64_t)-((int64_t)x)));
241#elif (LG_SIZEOF_PTR == 2)
242	return ((size_t)atomic_add_uint32((uint32_t *)p,
243	    (uint32_t)-((int32_t)x)));
244#endif
245}
246
247/******************************************************************************/
248/* unsigned operations. */
249JEMALLOC_INLINE unsigned
250atomic_add_u(unsigned *p, unsigned x)
251{
252
253#if (LG_SIZEOF_INT == 3)
254	return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
255#elif (LG_SIZEOF_INT == 2)
256	return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
257#endif
258}
259
260JEMALLOC_INLINE unsigned
261atomic_sub_u(unsigned *p, unsigned x)
262{
263
264#if (LG_SIZEOF_INT == 3)
265	return ((unsigned)atomic_add_uint64((uint64_t *)p,
266	    (uint64_t)-((int64_t)x)));
267#elif (LG_SIZEOF_INT == 2)
268	return ((unsigned)atomic_add_uint32((uint32_t *)p,
269	    (uint32_t)-((int32_t)x)));
270#endif
271}
272/******************************************************************************/
273#endif
274
275#endif /* JEMALLOC_H_INLINES */
276/******************************************************************************/
277