1 /*-
2 * Copyright (c) 2011 Ed Schouten <ed@FreeBSD.org>
3 * David Chisnall <theraven@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 * $FreeBSD$
28 */
29
30 #pragma once
31
32 #include <sys/cdefs.h>
33 #include <sys/types.h>
34 #include <stdbool.h>
35
36 /*
37 * C: Do it ourselves.
38 * Note that the runtime representation defined here should be compatible
39 * with the C++ one, i.e. an _Atomic(T) needs to contain the same
40 * bits as a T.
41 */
42
43 #include <stddef.h> /* For ptrdiff_t. */
44 #include <stdint.h>
45 // Include uchar.h only when available. Bionic's stdatomic.h is also used for
46 // the host (via a copy in prebuilts/clang) and uchar.h is not available in the
47 // glibc used for the host.
48 #if defined(__BIONIC__)
49 # include <uchar.h> /* For char16_t and char32_t. */
50 #endif
51
52 /*
53 * 7.17.1 Atomic lock-free macros.
54 */
55
56 #ifdef __GCC_ATOMIC_BOOL_LOCK_FREE
57 #define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE
58 #endif
59 #ifdef __GCC_ATOMIC_CHAR_LOCK_FREE
60 #define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE
61 #endif
62 #ifdef __GCC_ATOMIC_CHAR16_T_LOCK_FREE
63 #define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE
64 #endif
65 #ifdef __GCC_ATOMIC_CHAR32_T_LOCK_FREE
66 #define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE
67 #endif
68 #ifdef __GCC_ATOMIC_WCHAR_T_LOCK_FREE
69 #define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE
70 #endif
71 #ifdef __GCC_ATOMIC_SHORT_LOCK_FREE
72 #define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE
73 #endif
74 #ifdef __GCC_ATOMIC_INT_LOCK_FREE
75 #define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE
76 #endif
77 #ifdef __GCC_ATOMIC_LONG_LOCK_FREE
78 #define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE
79 #endif
80 #ifdef __GCC_ATOMIC_LLONG_LOCK_FREE
81 #define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE
82 #endif
83 #ifdef __GCC_ATOMIC_POINTER_LOCK_FREE
84 #define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE
85 #endif
86
87 /*
88 * 7.17.2 Initialization.
89 */
90
91 #define ATOMIC_VAR_INIT(value) (value)
92 #define atomic_init(obj, value) __c11_atomic_init(obj, value)
93
94 /*
95 * Clang and recent GCC both provide predefined macros for the memory
96 * orderings. If we are using a compiler that doesn't define them, use the
97 * clang values - these will be ignored in the fallback path.
98 */
99
100 #ifndef __ATOMIC_RELAXED
101 #define __ATOMIC_RELAXED 0
102 #endif
103 #ifndef __ATOMIC_CONSUME
104 #define __ATOMIC_CONSUME 1
105 #endif
106 #ifndef __ATOMIC_ACQUIRE
107 #define __ATOMIC_ACQUIRE 2
108 #endif
109 #ifndef __ATOMIC_RELEASE
110 #define __ATOMIC_RELEASE 3
111 #endif
112 #ifndef __ATOMIC_ACQ_REL
113 #define __ATOMIC_ACQ_REL 4
114 #endif
115 #ifndef __ATOMIC_SEQ_CST
116 #define __ATOMIC_SEQ_CST 5
117 #endif
118
119 /*
120 * 7.17.3 Order and consistency.
121 *
122 * The memory_order_* constants that denote the barrier behaviour of the
123 * atomic operations.
124 * The enum values must be identical to those used by the
125 * C++ <atomic> header.
126 */
127
128 typedef enum {
129 memory_order_relaxed = __ATOMIC_RELAXED,
130 memory_order_consume = __ATOMIC_CONSUME,
131 memory_order_acquire = __ATOMIC_ACQUIRE,
132 memory_order_release = __ATOMIC_RELEASE,
133 memory_order_acq_rel = __ATOMIC_ACQ_REL,
134 memory_order_seq_cst = __ATOMIC_SEQ_CST
135 } memory_order;
136
137 /*
138 * 7.17.4 Fences.
139 */
140
atomic_thread_fence(memory_order __order)141 static __inline void atomic_thread_fence(memory_order __order __attribute__((unused))) {
142 __c11_atomic_thread_fence(__order);
143 }
144
atomic_signal_fence(memory_order __order)145 static __inline void atomic_signal_fence(memory_order __order __attribute__((unused))) {
146 __c11_atomic_signal_fence(__order);
147 }
148
149 /*
150 * 7.17.5 Lock-free property.
151 */
152
153 #define atomic_is_lock_free(obj) __c11_atomic_is_lock_free(sizeof(*(obj)))
154
155 /*
156 * 7.17.6 Atomic integer types.
157 */
158
159 typedef _Atomic(bool) atomic_bool;
160 typedef _Atomic(char) atomic_char;
161 typedef _Atomic(signed char) atomic_schar;
162 typedef _Atomic(unsigned char) atomic_uchar;
163 typedef _Atomic(short) atomic_short;
164 typedef _Atomic(unsigned short) atomic_ushort;
165 typedef _Atomic(int) atomic_int;
166 typedef _Atomic(unsigned int) atomic_uint;
167 typedef _Atomic(long) atomic_long;
168 typedef _Atomic(unsigned long) atomic_ulong;
169 typedef _Atomic(long long) atomic_llong;
170 typedef _Atomic(unsigned long long) atomic_ullong;
171 #if defined(__BIONIC__) || (defined(__cplusplus) && __cplusplus >= 201103L)
172 typedef _Atomic(char16_t) atomic_char16_t;
173 typedef _Atomic(char32_t) atomic_char32_t;
174 #endif
175 typedef _Atomic(wchar_t) atomic_wchar_t;
176 typedef _Atomic(int_least8_t) atomic_int_least8_t;
177 typedef _Atomic(uint_least8_t) atomic_uint_least8_t;
178 typedef _Atomic(int_least16_t) atomic_int_least16_t;
179 typedef _Atomic(uint_least16_t) atomic_uint_least16_t;
180 typedef _Atomic(int_least32_t) atomic_int_least32_t;
181 typedef _Atomic(uint_least32_t) atomic_uint_least32_t;
182 typedef _Atomic(int_least64_t) atomic_int_least64_t;
183 typedef _Atomic(uint_least64_t) atomic_uint_least64_t;
184 typedef _Atomic(int_fast8_t) atomic_int_fast8_t;
185 typedef _Atomic(uint_fast8_t) atomic_uint_fast8_t;
186 typedef _Atomic(int_fast16_t) atomic_int_fast16_t;
187 typedef _Atomic(uint_fast16_t) atomic_uint_fast16_t;
188 typedef _Atomic(int_fast32_t) atomic_int_fast32_t;
189 typedef _Atomic(uint_fast32_t) atomic_uint_fast32_t;
190 typedef _Atomic(int_fast64_t) atomic_int_fast64_t;
191 typedef _Atomic(uint_fast64_t) atomic_uint_fast64_t;
192 typedef _Atomic(intptr_t) atomic_intptr_t;
193 typedef _Atomic(uintptr_t) atomic_uintptr_t;
194 typedef _Atomic(size_t) atomic_size_t;
195 typedef _Atomic(ptrdiff_t) atomic_ptrdiff_t;
196 typedef _Atomic(intmax_t) atomic_intmax_t;
197 typedef _Atomic(uintmax_t) atomic_uintmax_t;
198
199 /*
200 * 7.17.7 Operations on atomic types.
201 */
202
203 /*
204 * Compiler-specific operations.
205 */
206
207 #define atomic_compare_exchange_strong_explicit(object, expected, \
208 desired, success, failure) \
209 __c11_atomic_compare_exchange_strong(object, expected, desired, \
210 success, failure)
211 #define atomic_compare_exchange_weak_explicit(object, expected, \
212 desired, success, failure) \
213 __c11_atomic_compare_exchange_weak(object, expected, desired, \
214 success, failure)
215 #define atomic_exchange_explicit(object, desired, order) \
216 __c11_atomic_exchange(object, desired, order)
217 #define atomic_fetch_add_explicit(object, operand, order) \
218 __c11_atomic_fetch_add(object, operand, order)
219 #define atomic_fetch_and_explicit(object, operand, order) \
220 __c11_atomic_fetch_and(object, operand, order)
221 #define atomic_fetch_or_explicit(object, operand, order) \
222 __c11_atomic_fetch_or(object, operand, order)
223 #define atomic_fetch_sub_explicit(object, operand, order) \
224 __c11_atomic_fetch_sub(object, operand, order)
225 #define atomic_fetch_xor_explicit(object, operand, order) \
226 __c11_atomic_fetch_xor(object, operand, order)
227 #define atomic_load_explicit(object, order) \
228 __c11_atomic_load(object, order)
229 #define atomic_store_explicit(object, desired, order) \
230 __c11_atomic_store(object, desired, order)
231
232 /*
233 * Convenience functions.
234 */
235
236 #define atomic_compare_exchange_strong(object, expected, desired) \
237 atomic_compare_exchange_strong_explicit(object, expected, \
238 desired, memory_order_seq_cst, memory_order_seq_cst)
239 #define atomic_compare_exchange_weak(object, expected, desired) \
240 atomic_compare_exchange_weak_explicit(object, expected, \
241 desired, memory_order_seq_cst, memory_order_seq_cst)
242 #define atomic_exchange(object, desired) \
243 atomic_exchange_explicit(object, desired, memory_order_seq_cst)
244 #define atomic_fetch_add(object, operand) \
245 atomic_fetch_add_explicit(object, operand, memory_order_seq_cst)
246 #define atomic_fetch_and(object, operand) \
247 atomic_fetch_and_explicit(object, operand, memory_order_seq_cst)
248 #define atomic_fetch_or(object, operand) \
249 atomic_fetch_or_explicit(object, operand, memory_order_seq_cst)
250 #define atomic_fetch_sub(object, operand) \
251 atomic_fetch_sub_explicit(object, operand, memory_order_seq_cst)
252 #define atomic_fetch_xor(object, operand) \
253 atomic_fetch_xor_explicit(object, operand, memory_order_seq_cst)
254 #define atomic_load(object) \
255 atomic_load_explicit(object, memory_order_seq_cst)
256 #define atomic_store(object, desired) \
257 atomic_store_explicit(object, desired, memory_order_seq_cst)
258
259 /*
260 * 7.17.8 Atomic flag type and operations.
261 *
262 * XXX: Assume atomic_bool can be used as an atomic_flag. Is there some
263 * kind of compiler built-in type we could use?
264 */
265
266 typedef struct {
267 atomic_bool __flag;
268 } atomic_flag;
269
270 #define ATOMIC_FLAG_INIT { ATOMIC_VAR_INIT(false) }
271
atomic_flag_test_and_set_explicit(volatile atomic_flag * __object,memory_order __order)272 static __inline bool atomic_flag_test_and_set_explicit(volatile atomic_flag *__object, memory_order __order) {
273 return (atomic_exchange_explicit(&__object->__flag, 1, __order));
274 }
275
atomic_flag_clear_explicit(volatile atomic_flag * __object,memory_order __order)276 static __inline void atomic_flag_clear_explicit(volatile atomic_flag *__object, memory_order __order) {
277 atomic_store_explicit(&__object->__flag, 0, __order);
278 }
279
atomic_flag_test_and_set(volatile atomic_flag * __object)280 static __inline bool atomic_flag_test_and_set(volatile atomic_flag *__object) {
281 return (atomic_flag_test_and_set_explicit(__object, memory_order_seq_cst));
282 }
283
atomic_flag_clear(volatile atomic_flag * __object)284 static __inline void atomic_flag_clear(volatile atomic_flag *__object) {
285 atomic_flag_clear_explicit(__object, memory_order_seq_cst);
286 }
287