1 /**
2 * Many similar implementations exist. See for example libwsbm
3 * or the linux kernel include/atomic.h
4 *
5 * No copyright claimed on this file.
6 *
7 */
8
9 #ifndef U_ATOMIC_H
10 #define U_ATOMIC_H
11
12 #include "pipe/p_compiler.h"
13 #include "pipe/p_defines.h"
14
15 /* Favor OS-provided implementations.
16 *
17 * Where no OS-provided implementation is available, fall back to
18 * locally coded assembly, compiler intrinsic or ultimately a
19 * mutex-based implementation.
20 */
21 #if defined(PIPE_OS_SOLARIS)
22 #define PIPE_ATOMIC_OS_SOLARIS
23 #elif defined(PIPE_CC_MSVC)
24 #define PIPE_ATOMIC_MSVC_INTRINSIC
25 #elif (defined(PIPE_CC_MSVC) && defined(PIPE_ARCH_X86))
26 #define PIPE_ATOMIC_ASM_MSVC_X86
27 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86))
28 #define PIPE_ATOMIC_ASM_GCC_X86
29 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64))
30 #define PIPE_ATOMIC_ASM_GCC_X86_64
31 #elif defined(PIPE_CC_GCC) && (PIPE_CC_GCC_VERSION >= 401)
32 #define PIPE_ATOMIC_GCC_INTRINSIC
33 #else
34 #error "Unsupported platform"
35 #endif
36
37
38 #if defined(PIPE_ATOMIC_ASM_GCC_X86_64)
39 #define PIPE_ATOMIC "GCC x86_64 assembly"
40
41 #ifdef __cplusplus
42 extern "C" {
43 #endif
44
45 #define p_atomic_set(_v, _i) (*(_v) = (_i))
46 #define p_atomic_read(_v) (*(_v))
47
48 static INLINE boolean
p_atomic_dec_zero(int32_t * v)49 p_atomic_dec_zero(int32_t *v)
50 {
51 unsigned char c;
52
53 __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
54 ::"memory");
55
56 return c != 0;
57 }
58
59 static INLINE void
p_atomic_inc(int32_t * v)60 p_atomic_inc(int32_t *v)
61 {
62 __asm__ __volatile__("lock; incl %0":"+m"(*v));
63 }
64
65 static INLINE void
p_atomic_dec(int32_t * v)66 p_atomic_dec(int32_t *v)
67 {
68 __asm__ __volatile__("lock; decl %0":"+m"(*v));
69 }
70
71 static INLINE int32_t
p_atomic_cmpxchg(int32_t * v,int32_t old,int32_t _new)72 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
73 {
74 return __sync_val_compare_and_swap(v, old, _new);
75 }
76
77 #ifdef __cplusplus
78 }
79 #endif
80
81 #endif /* PIPE_ATOMIC_ASM_GCC_X86_64 */
82
83
84 #if defined(PIPE_ATOMIC_ASM_GCC_X86)
85
86 #define PIPE_ATOMIC "GCC x86 assembly"
87
88 #ifdef __cplusplus
89 extern "C" {
90 #endif
91
92 #define p_atomic_set(_v, _i) (*(_v) = (_i))
93 #define p_atomic_read(_v) (*(_v))
94
95 static INLINE boolean
p_atomic_dec_zero(int32_t * v)96 p_atomic_dec_zero(int32_t *v)
97 {
98 unsigned char c;
99
100 __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
101 ::"memory");
102
103 return c != 0;
104 }
105
106 static INLINE void
p_atomic_inc(int32_t * v)107 p_atomic_inc(int32_t *v)
108 {
109 __asm__ __volatile__("lock; incl %0":"+m"(*v));
110 }
111
112 static INLINE void
p_atomic_dec(int32_t * v)113 p_atomic_dec(int32_t *v)
114 {
115 __asm__ __volatile__("lock; decl %0":"+m"(*v));
116 }
117
118 static INLINE int32_t
p_atomic_cmpxchg(int32_t * v,int32_t old,int32_t _new)119 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
120 {
121 return __sync_val_compare_and_swap(v, old, _new);
122 }
123
124 #ifdef __cplusplus
125 }
126 #endif
127
128 #endif
129
130
131
132 /* Implementation using GCC-provided synchronization intrinsics
133 */
134 #if defined(PIPE_ATOMIC_GCC_INTRINSIC)
135
136 #define PIPE_ATOMIC "GCC Sync Intrinsics"
137
138 #ifdef __cplusplus
139 extern "C" {
140 #endif
141
142 #define p_atomic_set(_v, _i) (*(_v) = (_i))
143 #define p_atomic_read(_v) (*(_v))
144
145 static INLINE boolean
p_atomic_dec_zero(int32_t * v)146 p_atomic_dec_zero(int32_t *v)
147 {
148 return (__sync_sub_and_fetch(v, 1) == 0);
149 }
150
151 static INLINE void
p_atomic_inc(int32_t * v)152 p_atomic_inc(int32_t *v)
153 {
154 (void) __sync_add_and_fetch(v, 1);
155 }
156
157 static INLINE void
p_atomic_dec(int32_t * v)158 p_atomic_dec(int32_t *v)
159 {
160 (void) __sync_sub_and_fetch(v, 1);
161 }
162
163 static INLINE int32_t
p_atomic_cmpxchg(int32_t * v,int32_t old,int32_t _new)164 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
165 {
166 return __sync_val_compare_and_swap(v, old, _new);
167 }
168
169 #ifdef __cplusplus
170 }
171 #endif
172
173 #endif
174
175
176
177 /* Unlocked version for single threaded environments, such as some
178 * windows kernel modules.
179 */
180 #if defined(PIPE_ATOMIC_OS_UNLOCKED)
181
182 #define PIPE_ATOMIC "Unlocked"
183
184 #define p_atomic_set(_v, _i) (*(_v) = (_i))
185 #define p_atomic_read(_v) (*(_v))
186 #define p_atomic_dec_zero(_v) ((boolean) --(*(_v)))
187 #define p_atomic_inc(_v) ((void) (*(_v))++)
188 #define p_atomic_dec(_v) ((void) (*(_v))--)
189 #define p_atomic_cmpxchg(_v, old, _new) (*(_v) == old ? *(_v) = (_new) : *(_v))
190
191 #endif
192
193
194 /* Locally coded assembly for MSVC on x86:
195 */
196 #if defined(PIPE_ATOMIC_ASM_MSVC_X86)
197
198 #define PIPE_ATOMIC "MSVC x86 assembly"
199
200 #ifdef __cplusplus
201 extern "C" {
202 #endif
203
204 #define p_atomic_set(_v, _i) (*(_v) = (_i))
205 #define p_atomic_read(_v) (*(_v))
206
207 static INLINE boolean
p_atomic_dec_zero(int32_t * v)208 p_atomic_dec_zero(int32_t *v)
209 {
210 unsigned char c;
211
212 __asm {
213 mov eax, [v]
214 lock dec dword ptr [eax]
215 sete byte ptr [c]
216 }
217
218 return c != 0;
219 }
220
221 static INLINE void
p_atomic_inc(int32_t * v)222 p_atomic_inc(int32_t *v)
223 {
224 __asm {
225 mov eax, [v]
226 lock inc dword ptr [eax]
227 }
228 }
229
230 static INLINE void
p_atomic_dec(int32_t * v)231 p_atomic_dec(int32_t *v)
232 {
233 __asm {
234 mov eax, [v]
235 lock dec dword ptr [eax]
236 }
237 }
238
239 static INLINE int32_t
p_atomic_cmpxchg(int32_t * v,int32_t old,int32_t _new)240 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
241 {
242 int32_t orig;
243
244 __asm {
245 mov ecx, [v]
246 mov eax, [old]
247 mov edx, [_new]
248 lock cmpxchg [ecx], edx
249 mov [orig], eax
250 }
251
252 return orig;
253 }
254
255 #ifdef __cplusplus
256 }
257 #endif
258
259 #endif
260
261
262 #if defined(PIPE_ATOMIC_MSVC_INTRINSIC)
263
264 #define PIPE_ATOMIC "MSVC Intrinsics"
265
266 #include <intrin.h>
267
268 #pragma intrinsic(_InterlockedIncrement)
269 #pragma intrinsic(_InterlockedDecrement)
270 #pragma intrinsic(_InterlockedCompareExchange)
271
272 #ifdef __cplusplus
273 extern "C" {
274 #endif
275
276 #define p_atomic_set(_v, _i) (*(_v) = (_i))
277 #define p_atomic_read(_v) (*(_v))
278
279 static INLINE boolean
p_atomic_dec_zero(int32_t * v)280 p_atomic_dec_zero(int32_t *v)
281 {
282 return _InterlockedDecrement((long *)v) == 0;
283 }
284
285 static INLINE void
p_atomic_inc(int32_t * v)286 p_atomic_inc(int32_t *v)
287 {
288 _InterlockedIncrement((long *)v);
289 }
290
291 static INLINE void
p_atomic_dec(int32_t * v)292 p_atomic_dec(int32_t *v)
293 {
294 _InterlockedDecrement((long *)v);
295 }
296
297 static INLINE int32_t
p_atomic_cmpxchg(int32_t * v,int32_t old,int32_t _new)298 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
299 {
300 return _InterlockedCompareExchange((long *)v, _new, old);
301 }
302
303 #ifdef __cplusplus
304 }
305 #endif
306
307 #endif
308
309 #if defined(PIPE_ATOMIC_OS_SOLARIS)
310
311 #define PIPE_ATOMIC "Solaris OS atomic functions"
312
313 #include <atomic.h>
314
315 #ifdef __cplusplus
316 extern "C" {
317 #endif
318
319 #define p_atomic_set(_v, _i) (*(_v) = (_i))
320 #define p_atomic_read(_v) (*(_v))
321
322 static INLINE boolean
p_atomic_dec_zero(int32_t * v)323 p_atomic_dec_zero(int32_t *v)
324 {
325 uint32_t n = atomic_dec_32_nv((uint32_t *) v);
326
327 return n != 0;
328 }
329
330 #define p_atomic_inc(_v) atomic_inc_32((uint32_t *) _v)
331 #define p_atomic_dec(_v) atomic_dec_32((uint32_t *) _v)
332
333 #define p_atomic_cmpxchg(_v, _old, _new) \
334 atomic_cas_32( (uint32_t *) _v, (uint32_t) _old, (uint32_t) _new)
335
336 #ifdef __cplusplus
337 }
338 #endif
339
340 #endif
341
342
343 #ifndef PIPE_ATOMIC
344 #error "No pipe_atomic implementation selected"
345 #endif
346
347
348
349 #endif /* U_ATOMIC_H */
350