1 //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file is a part of ThreadSanitizer/AddressSanitizer runtime.
11 // Not intended for direct inclusion. Include sanitizer_atomic.h.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #ifndef SANITIZER_ATOMIC_MSVC_H
16 #define SANITIZER_ATOMIC_MSVC_H
17
18 extern "C" void _ReadWriteBarrier();
19 #pragma intrinsic(_ReadWriteBarrier)
20 extern "C" void _mm_mfence();
21 #pragma intrinsic(_mm_mfence)
22 extern "C" void _mm_pause();
23 #pragma intrinsic(_mm_pause)
24 extern "C" char _InterlockedExchange8( // NOLINT
25 char volatile *Addend, char Value); // NOLINT
26 #pragma intrinsic(_InterlockedExchange8)
27 extern "C" short _InterlockedExchange16( // NOLINT
28 short volatile *Addend, short Value); // NOLINT
29 #pragma intrinsic(_InterlockedExchange16)
30 extern "C" long _InterlockedExchange( // NOLINT
31 long volatile *Addend, long Value); // NOLINT
32 #pragma intrinsic(_InterlockedExchange)
33 extern "C" long _InterlockedExchangeAdd( // NOLINT
34 long volatile * Addend, long Value); // NOLINT
35 #pragma intrinsic(_InterlockedExchangeAdd)
36 extern "C" short _InterlockedCompareExchange16( // NOLINT
37 short volatile *Destination, // NOLINT
38 short Exchange, short Comparand); // NOLINT
39 #pragma intrinsic(_InterlockedCompareExchange16)
40 extern "C"
41 long long _InterlockedCompareExchange64( // NOLINT
42 long long volatile *Destination, // NOLINT
43 long long Exchange, long long Comparand); // NOLINT
44 #pragma intrinsic(_InterlockedCompareExchange64)
45 extern "C" void *_InterlockedCompareExchangePointer(
46 void *volatile *Destination,
47 void *Exchange, void *Comparand);
48 #pragma intrinsic(_InterlockedCompareExchangePointer)
49 extern "C"
50 long __cdecl _InterlockedCompareExchange( // NOLINT
51 long volatile *Destination, // NOLINT
52 long Exchange, long Comparand); // NOLINT
53 #pragma intrinsic(_InterlockedCompareExchange)
54
55 #ifdef _WIN64
56 extern "C" long long _InterlockedExchangeAdd64( // NOLINT
57 long long volatile * Addend, long long Value); // NOLINT
58 #pragma intrinsic(_InterlockedExchangeAdd64)
59 #endif
60
61 namespace __sanitizer {
62
atomic_signal_fence(memory_order)63 INLINE void atomic_signal_fence(memory_order) {
64 _ReadWriteBarrier();
65 }
66
atomic_thread_fence(memory_order)67 INLINE void atomic_thread_fence(memory_order) {
68 _mm_mfence();
69 }
70
proc_yield(int cnt)71 INLINE void proc_yield(int cnt) {
72 for (int i = 0; i < cnt; i++)
73 _mm_pause();
74 }
75
76 template<typename T>
atomic_load(const volatile T * a,memory_order mo)77 INLINE typename T::Type atomic_load(
78 const volatile T *a, memory_order mo) {
79 DCHECK(mo & (memory_order_relaxed | memory_order_consume
80 | memory_order_acquire | memory_order_seq_cst));
81 DCHECK(!((uptr)a % sizeof(*a)));
82 typename T::Type v;
83 // FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
84 if (mo == memory_order_relaxed) {
85 v = a->val_dont_use;
86 } else {
87 atomic_signal_fence(memory_order_seq_cst);
88 v = a->val_dont_use;
89 atomic_signal_fence(memory_order_seq_cst);
90 }
91 return v;
92 }
93
94 template<typename T>
atomic_store(volatile T * a,typename T::Type v,memory_order mo)95 INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
96 DCHECK(mo & (memory_order_relaxed | memory_order_release
97 | memory_order_seq_cst));
98 DCHECK(!((uptr)a % sizeof(*a)));
99 // FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
100 if (mo == memory_order_relaxed) {
101 a->val_dont_use = v;
102 } else {
103 atomic_signal_fence(memory_order_seq_cst);
104 a->val_dont_use = v;
105 atomic_signal_fence(memory_order_seq_cst);
106 }
107 if (mo == memory_order_seq_cst)
108 atomic_thread_fence(memory_order_seq_cst);
109 }
110
atomic_fetch_add(volatile atomic_uint32_t * a,u32 v,memory_order mo)111 INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
112 u32 v, memory_order mo) {
113 (void)mo;
114 DCHECK(!((uptr)a % sizeof(*a)));
115 return (u32)_InterlockedExchangeAdd(
116 (volatile long*)&a->val_dont_use, (long)v); // NOLINT
117 }
118
atomic_fetch_add(volatile atomic_uintptr_t * a,uptr v,memory_order mo)119 INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
120 uptr v, memory_order mo) {
121 (void)mo;
122 DCHECK(!((uptr)a % sizeof(*a)));
123 #ifdef _WIN64
124 return (uptr)_InterlockedExchangeAdd64(
125 (volatile long long*)&a->val_dont_use, (long long)v); // NOLINT
126 #else
127 return (uptr)_InterlockedExchangeAdd(
128 (volatile long*)&a->val_dont_use, (long)v); // NOLINT
129 #endif
130 }
131
atomic_fetch_sub(volatile atomic_uint32_t * a,u32 v,memory_order mo)132 INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
133 u32 v, memory_order mo) {
134 (void)mo;
135 DCHECK(!((uptr)a % sizeof(*a)));
136 return (u32)_InterlockedExchangeAdd(
137 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT
138 }
139
atomic_fetch_sub(volatile atomic_uintptr_t * a,uptr v,memory_order mo)140 INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
141 uptr v, memory_order mo) {
142 (void)mo;
143 DCHECK(!((uptr)a % sizeof(*a)));
144 #ifdef _WIN64
145 return (uptr)_InterlockedExchangeAdd64(
146 (volatile long long*)&a->val_dont_use, -(long long)v); // NOLINT
147 #else
148 return (uptr)_InterlockedExchangeAdd(
149 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT
150 #endif
151 }
152
atomic_exchange(volatile atomic_uint8_t * a,u8 v,memory_order mo)153 INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
154 u8 v, memory_order mo) {
155 (void)mo;
156 DCHECK(!((uptr)a % sizeof(*a)));
157 return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v);
158 }
159
atomic_exchange(volatile atomic_uint16_t * a,u16 v,memory_order mo)160 INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
161 u16 v, memory_order mo) {
162 (void)mo;
163 DCHECK(!((uptr)a % sizeof(*a)));
164 return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v);
165 }
166
atomic_exchange(volatile atomic_uint32_t * a,u32 v,memory_order mo)167 INLINE u32 atomic_exchange(volatile atomic_uint32_t *a,
168 u32 v, memory_order mo) {
169 (void)mo;
170 DCHECK(!((uptr)a % sizeof(*a)));
171 return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v);
172 }
173
174 #ifndef _WIN64
175
atomic_compare_exchange_strong(volatile atomic_uint8_t * a,u8 * cmp,u8 xchgv,memory_order mo)176 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
177 u8 *cmp,
178 u8 xchgv,
179 memory_order mo) {
180 (void)mo;
181 DCHECK(!((uptr)a % sizeof(*a)));
182 u8 cmpv = *cmp;
183 u8 prev;
184 __asm {
185 mov al, cmpv
186 mov ecx, a
187 mov dl, xchgv
188 lock cmpxchg [ecx], dl
189 mov prev, al
190 }
191 if (prev == cmpv)
192 return true;
193 *cmp = prev;
194 return false;
195 }
196
197 #endif
198
atomic_compare_exchange_strong(volatile atomic_uintptr_t * a,uptr * cmp,uptr xchg,memory_order mo)199 INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
200 uptr *cmp,
201 uptr xchg,
202 memory_order mo) {
203 uptr cmpv = *cmp;
204 uptr prev = (uptr)_InterlockedCompareExchangePointer(
205 (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv);
206 if (prev == cmpv)
207 return true;
208 *cmp = prev;
209 return false;
210 }
211
atomic_compare_exchange_strong(volatile atomic_uint16_t * a,u16 * cmp,u16 xchg,memory_order mo)212 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
213 u16 *cmp,
214 u16 xchg,
215 memory_order mo) {
216 u16 cmpv = *cmp;
217 u16 prev = (u16)_InterlockedCompareExchange16(
218 (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv);
219 if (prev == cmpv)
220 return true;
221 *cmp = prev;
222 return false;
223 }
224
atomic_compare_exchange_strong(volatile atomic_uint32_t * a,u32 * cmp,u32 xchg,memory_order mo)225 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
226 u32 *cmp,
227 u32 xchg,
228 memory_order mo) {
229 u32 cmpv = *cmp;
230 u32 prev = (u32)_InterlockedCompareExchange(
231 (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv);
232 if (prev == cmpv)
233 return true;
234 *cmp = prev;
235 return false;
236 }
237
atomic_compare_exchange_strong(volatile atomic_uint64_t * a,u64 * cmp,u64 xchg,memory_order mo)238 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
239 u64 *cmp,
240 u64 xchg,
241 memory_order mo) {
242 u64 cmpv = *cmp;
243 u64 prev = (u64)_InterlockedCompareExchange64(
244 (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv);
245 if (prev == cmpv)
246 return true;
247 *cmp = prev;
248 return false;
249 }
250
251 template<typename T>
atomic_compare_exchange_weak(volatile T * a,typename T::Type * cmp,typename T::Type xchg,memory_order mo)252 INLINE bool atomic_compare_exchange_weak(volatile T *a,
253 typename T::Type *cmp,
254 typename T::Type xchg,
255 memory_order mo) {
256 return atomic_compare_exchange_strong(a, cmp, xchg, mo);
257 }
258
259 } // namespace __sanitizer
260
261 #endif // SANITIZER_ATOMIC_CLANG_H
262