1/******************************************************************************/
2#ifdef JEMALLOC_H_TYPES
3
4#endif /* JEMALLOC_H_TYPES */
5/******************************************************************************/
6#ifdef JEMALLOC_H_STRUCTS
7
8#endif /* JEMALLOC_H_STRUCTS */
9/******************************************************************************/
10#ifdef JEMALLOC_H_EXTERNS
11
12#define	atomic_read_uint64(p)	atomic_add_uint64(p, 0)
13#define	atomic_read_uint32(p)	atomic_add_uint32(p, 0)
14#define	atomic_read_z(p)	atomic_add_z(p, 0)
15#define	atomic_read_u(p)	atomic_add_u(p, 0)
16
17#endif /* JEMALLOC_H_EXTERNS */
18/******************************************************************************/
19#ifdef JEMALLOC_H_INLINES
20
21#ifndef JEMALLOC_ENABLE_INLINE
22uint64_t	atomic_add_uint64(uint64_t *p, uint64_t x);
23uint64_t	atomic_sub_uint64(uint64_t *p, uint64_t x);
24uint32_t	atomic_add_uint32(uint32_t *p, uint32_t x);
25uint32_t	atomic_sub_uint32(uint32_t *p, uint32_t x);
26size_t	atomic_add_z(size_t *p, size_t x);
27size_t	atomic_sub_z(size_t *p, size_t x);
28unsigned	atomic_add_u(unsigned *p, unsigned x);
29unsigned	atomic_sub_u(unsigned *p, unsigned x);
30#endif
31
32#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_))
33/******************************************************************************/
34/* 64-bit operations. */
35#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
36#  ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
37JEMALLOC_INLINE uint64_t
38atomic_add_uint64(uint64_t *p, uint64_t x)
39{
40
41	return (__sync_add_and_fetch(p, x));
42}
43
44JEMALLOC_INLINE uint64_t
45atomic_sub_uint64(uint64_t *p, uint64_t x)
46{
47
48	return (__sync_sub_and_fetch(p, x));
49}
50#elif (defined(_MSC_VER))
51JEMALLOC_INLINE uint64_t
52atomic_add_uint64(uint64_t *p, uint64_t x)
53{
54
55	return (InterlockedExchangeAdd64(p, x));
56}
57
58JEMALLOC_INLINE uint64_t
59atomic_sub_uint64(uint64_t *p, uint64_t x)
60{
61
62	return (InterlockedExchangeAdd64(p, -((int64_t)x)));
63}
64#elif (defined(JEMALLOC_OSATOMIC))
65JEMALLOC_INLINE uint64_t
66atomic_add_uint64(uint64_t *p, uint64_t x)
67{
68
69	return (OSAtomicAdd64((int64_t)x, (int64_t *)p));
70}
71
72JEMALLOC_INLINE uint64_t
73atomic_sub_uint64(uint64_t *p, uint64_t x)
74{
75
76	return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
77}
78#  elif (defined(__amd64__) || defined(__x86_64__))
79JEMALLOC_INLINE uint64_t
80atomic_add_uint64(uint64_t *p, uint64_t x)
81{
82
83	asm volatile (
84	    "lock; xaddq %0, %1;"
85	    : "+r" (x), "=m" (*p) /* Outputs. */
86	    : "m" (*p) /* Inputs. */
87	    );
88
89	return (x);
90}
91
92JEMALLOC_INLINE uint64_t
93atomic_sub_uint64(uint64_t *p, uint64_t x)
94{
95
96	x = (uint64_t)(-(int64_t)x);
97	asm volatile (
98	    "lock; xaddq %0, %1;"
99	    : "+r" (x), "=m" (*p) /* Outputs. */
100	    : "m" (*p) /* Inputs. */
101	    );
102
103	return (x);
104}
105#  elif (defined(JEMALLOC_ATOMIC9))
106JEMALLOC_INLINE uint64_t
107atomic_add_uint64(uint64_t *p, uint64_t x)
108{
109
110	/*
111	 * atomic_fetchadd_64() doesn't exist, but we only ever use this
112	 * function on LP64 systems, so atomic_fetchadd_long() will do.
113	 */
114	assert(sizeof(uint64_t) == sizeof(unsigned long));
115
116	return (atomic_fetchadd_long(p, (unsigned long)x) + x);
117}
118
119JEMALLOC_INLINE uint64_t
120atomic_sub_uint64(uint64_t *p, uint64_t x)
121{
122
123	assert(sizeof(uint64_t) == sizeof(unsigned long));
124
125	return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x);
126}
127#  elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
128JEMALLOC_INLINE uint64_t
129atomic_add_uint64(uint64_t *p, uint64_t x)
130{
131
132	return (__sync_add_and_fetch(p, x));
133}
134
135JEMALLOC_INLINE uint64_t
136atomic_sub_uint64(uint64_t *p, uint64_t x)
137{
138
139	return (__sync_sub_and_fetch(p, x));
140}
141#  else
142#    error "Missing implementation for 64-bit atomic operations"
143#  endif
144#endif
145
146/******************************************************************************/
147/* 32-bit operations. */
148#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4
149JEMALLOC_INLINE uint32_t
150atomic_add_uint32(uint32_t *p, uint32_t x)
151{
152
153	return (__sync_add_and_fetch(p, x));
154}
155
156JEMALLOC_INLINE uint32_t
157atomic_sub_uint32(uint32_t *p, uint32_t x)
158{
159
160	return (__sync_sub_and_fetch(p, x));
161}
162#elif (defined(_MSC_VER))
163JEMALLOC_INLINE uint32_t
164atomic_add_uint32(uint32_t *p, uint32_t x)
165{
166
167	return (InterlockedExchangeAdd(p, x));
168}
169
170JEMALLOC_INLINE uint32_t
171atomic_sub_uint32(uint32_t *p, uint32_t x)
172{
173
174	return (InterlockedExchangeAdd(p, -((int32_t)x)));
175}
176#elif (defined(JEMALLOC_OSATOMIC))
177JEMALLOC_INLINE uint32_t
178atomic_add_uint32(uint32_t *p, uint32_t x)
179{
180
181	return (OSAtomicAdd32((int32_t)x, (int32_t *)p));
182}
183
184JEMALLOC_INLINE uint32_t
185atomic_sub_uint32(uint32_t *p, uint32_t x)
186{
187
188	return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p));
189}
190#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
191JEMALLOC_INLINE uint32_t
192atomic_add_uint32(uint32_t *p, uint32_t x)
193{
194
195	asm volatile (
196	    "lock; xaddl %0, %1;"
197	    : "+r" (x), "=m" (*p) /* Outputs. */
198	    : "m" (*p) /* Inputs. */
199	    );
200
201	return (x);
202}
203
204JEMALLOC_INLINE uint32_t
205atomic_sub_uint32(uint32_t *p, uint32_t x)
206{
207
208	x = (uint32_t)(-(int32_t)x);
209	asm volatile (
210	    "lock; xaddl %0, %1;"
211	    : "+r" (x), "=m" (*p) /* Outputs. */
212	    : "m" (*p) /* Inputs. */
213	    );
214
215	return (x);
216}
217#elif (defined(JEMALLOC_ATOMIC9))
218JEMALLOC_INLINE uint32_t
219atomic_add_uint32(uint32_t *p, uint32_t x)
220{
221
222	return (atomic_fetchadd_32(p, x) + x);
223}
224
225JEMALLOC_INLINE uint32_t
226atomic_sub_uint32(uint32_t *p, uint32_t x)
227{
228
229	return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x);
230}
231#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
232JEMALLOC_INLINE uint32_t
233atomic_add_uint32(uint32_t *p, uint32_t x)
234{
235
236	return (__sync_add_and_fetch(p, x));
237}
238
239JEMALLOC_INLINE uint32_t
240atomic_sub_uint32(uint32_t *p, uint32_t x)
241{
242
243	return (__sync_sub_and_fetch(p, x));
244}
245#else
246#  error "Missing implementation for 32-bit atomic operations"
247#endif
248
249/******************************************************************************/
250/* size_t operations. */
251JEMALLOC_INLINE size_t
252atomic_add_z(size_t *p, size_t x)
253{
254
255#if (LG_SIZEOF_PTR == 3)
256	return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
257#elif (LG_SIZEOF_PTR == 2)
258	return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
259#endif
260}
261
262JEMALLOC_INLINE size_t
263atomic_sub_z(size_t *p, size_t x)
264{
265
266#if (LG_SIZEOF_PTR == 3)
267	return ((size_t)atomic_add_uint64((uint64_t *)p,
268	    (uint64_t)-((int64_t)x)));
269#elif (LG_SIZEOF_PTR == 2)
270	return ((size_t)atomic_add_uint32((uint32_t *)p,
271	    (uint32_t)-((int32_t)x)));
272#endif
273}
274
275/******************************************************************************/
276/* unsigned operations. */
277JEMALLOC_INLINE unsigned
278atomic_add_u(unsigned *p, unsigned x)
279{
280
281#if (LG_SIZEOF_INT == 3)
282	return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
283#elif (LG_SIZEOF_INT == 2)
284	return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
285#endif
286}
287
288JEMALLOC_INLINE unsigned
289atomic_sub_u(unsigned *p, unsigned x)
290{
291
292#if (LG_SIZEOF_INT == 3)
293	return ((unsigned)atomic_add_uint64((uint64_t *)p,
294	    (uint64_t)-((int64_t)x)));
295#elif (LG_SIZEOF_INT == 2)
296	return ((unsigned)atomic_add_uint32((uint32_t *)p,
297	    (uint32_t)-((int32_t)x)));
298#endif
299}
300/******************************************************************************/
301#endif
302
303#endif /* JEMALLOC_H_INLINES */
304/******************************************************************************/
305