1 /*===---- immintrin.h - Intel intrinsics -----------------------------------===
2  *
3  * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4  * See https://llvm.org/LICENSE.txt for license information.
5  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6  *
7  *===-----------------------------------------------------------------------===
8  */
9 
10 #ifndef __IMMINTRIN_H
11 #define __IMMINTRIN_H
12 
13 #include <x86gprintrin.h>
14 
15 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
16     defined(__MMX__)
17 #include <mmintrin.h>
18 #endif
19 
20 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
21     defined(__SSE__)
22 #include <xmmintrin.h>
23 #endif
24 
25 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
26     defined(__SSE2__)
27 #include <emmintrin.h>
28 #endif
29 
30 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
31     defined(__SSE3__)
32 #include <pmmintrin.h>
33 #endif
34 
35 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
36     defined(__SSSE3__)
37 #include <tmmintrin.h>
38 #endif
39 
40 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
41     (defined(__SSE4_2__) || defined(__SSE4_1__))
42 #include <smmintrin.h>
43 #endif
44 
45 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
46     (defined(__AES__) || defined(__PCLMUL__))
47 #include <wmmintrin.h>
48 #endif
49 
50 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
51     defined(__CLFLUSHOPT__)
52 #include <clflushoptintrin.h>
53 #endif
54 
55 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
56     defined(__CLWB__)
57 #include <clwbintrin.h>
58 #endif
59 
60 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
61     defined(__AVX__)
62 #include <avxintrin.h>
63 #endif
64 
65 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
66     defined(__AVX2__)
67 #include <avx2intrin.h>
68 #endif
69 
70 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
71     defined(__F16C__)
72 #include <f16cintrin.h>
73 #endif
74 
75 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
76     defined(__VPCLMULQDQ__)
77 #include <vpclmulqdqintrin.h>
78 #endif
79 
80 /* No feature check desired due to internal checks */
81 #include <bmiintrin.h>
82 
83 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
84     defined(__BMI2__)
85 #include <bmi2intrin.h>
86 #endif
87 
88 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
89     defined(__LZCNT__)
90 #include <lzcntintrin.h>
91 #endif
92 
93 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
94     defined(__POPCNT__)
95 #include <popcntintrin.h>
96 #endif
97 
98 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
99     defined(__FMA__)
100 #include <fmaintrin.h>
101 #endif
102 
103 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
104     defined(__AVX512F__)
105 #include <avx512fintrin.h>
106 #endif
107 
108 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
109     defined(__AVX512VL__)
110 #include <avx512vlintrin.h>
111 #endif
112 
113 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
114     defined(__AVX512BW__)
115 #include <avx512bwintrin.h>
116 #endif
117 
118 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
119     defined(__AVX512BITALG__)
120 #include <avx512bitalgintrin.h>
121 #endif
122 
123 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
124     defined(__AVX512CD__)
125 #include <avx512cdintrin.h>
126 #endif
127 
128 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
129     defined(__AVX512VPOPCNTDQ__)
130 #include <avx512vpopcntdqintrin.h>
131 #endif
132 
133 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
134     (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__))
135 #include <avx512vpopcntdqvlintrin.h>
136 #endif
137 
138 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
139     defined(__AVX512VNNI__)
140 #include <avx512vnniintrin.h>
141 #endif
142 
143 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
144     (defined(__AVX512VL__) && defined(__AVX512VNNI__))
145 #include <avx512vlvnniintrin.h>
146 #endif
147 
148 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
149     defined(__AVXVNNI__)
150 #include <avxvnniintrin.h>
151 #endif
152 
153 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
154     defined(__AVX512DQ__)
155 #include <avx512dqintrin.h>
156 #endif
157 
158 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
159     (defined(__AVX512VL__) && defined(__AVX512BITALG__))
160 #include <avx512vlbitalgintrin.h>
161 #endif
162 
163 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
164     (defined(__AVX512VL__) && defined(__AVX512BW__))
165 #include <avx512vlbwintrin.h>
166 #endif
167 
168 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
169     (defined(__AVX512VL__) && defined(__AVX512CD__))
170 #include <avx512vlcdintrin.h>
171 #endif
172 
173 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
174     (defined(__AVX512VL__) && defined(__AVX512DQ__))
175 #include <avx512vldqintrin.h>
176 #endif
177 
178 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
179     defined(__AVX512ER__)
180 #include <avx512erintrin.h>
181 #endif
182 
183 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
184     defined(__AVX512IFMA__)
185 #include <avx512ifmaintrin.h>
186 #endif
187 
188 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
189     (defined(__AVX512IFMA__) && defined(__AVX512VL__))
190 #include <avx512ifmavlintrin.h>
191 #endif
192 
193 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
194     defined(__AVX512VBMI__)
195 #include <avx512vbmiintrin.h>
196 #endif
197 
198 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
199     (defined(__AVX512VBMI__) && defined(__AVX512VL__))
200 #include <avx512vbmivlintrin.h>
201 #endif
202 
203 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
204     defined(__AVX512VBMI2__)
205 #include <avx512vbmi2intrin.h>
206 #endif
207 
208 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
209     (defined(__AVX512VBMI2__) && defined(__AVX512VL__))
210 #include <avx512vlvbmi2intrin.h>
211 #endif
212 
213 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
214     defined(__AVX512PF__)
215 #include <avx512pfintrin.h>
216 #endif
217 
218 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
219     defined(__AVX512BF16__)
220 #include <avx512bf16intrin.h>
221 #endif
222 
223 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
224     (defined(__AVX512VL__) && defined(__AVX512BF16__))
225 #include <avx512vlbf16intrin.h>
226 #endif
227 
228 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
229     defined(__PKU__)
230 #include <pkuintrin.h>
231 #endif
232 
233 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
234     defined(__VAES__)
235 #include <vaesintrin.h>
236 #endif
237 
238 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
239     defined(__GFNI__)
240 #include <gfniintrin.h>
241 #endif
242 
243 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
244     defined(__RDPID__)
245 /// Returns the value of the IA32_TSC_AUX MSR (0xc0000103).
246 ///
247 /// \headerfile <immintrin.h>
248 ///
249 /// This intrinsic corresponds to the <c> RDPID </c> instruction.
250 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid")))
_rdpid_u32(void)251 _rdpid_u32(void) {
252   return __builtin_ia32_rdpid();
253 }
254 #endif // __RDPID__
255 
256 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
257     defined(__RDRND__)
258 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
_rdrand16_step(unsigned short * __p)259 _rdrand16_step(unsigned short *__p)
260 {
261   return __builtin_ia32_rdrand16_step(__p);
262 }
263 
264 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
_rdrand32_step(unsigned int * __p)265 _rdrand32_step(unsigned int *__p)
266 {
267   return __builtin_ia32_rdrand32_step(__p);
268 }
269 
270 #ifdef __x86_64__
271 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
_rdrand64_step(unsigned long long * __p)272 _rdrand64_step(unsigned long long *__p)
273 {
274   return __builtin_ia32_rdrand64_step(__p);
275 }
276 #endif
277 #endif /* __RDRND__ */
278 
279 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
280     defined(__FSGSBASE__)
281 #ifdef __x86_64__
282 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_readfsbase_u32(void)283 _readfsbase_u32(void)
284 {
285   return __builtin_ia32_rdfsbase32();
286 }
287 
288 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_readfsbase_u64(void)289 _readfsbase_u64(void)
290 {
291   return __builtin_ia32_rdfsbase64();
292 }
293 
294 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_readgsbase_u32(void)295 _readgsbase_u32(void)
296 {
297   return __builtin_ia32_rdgsbase32();
298 }
299 
300 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_readgsbase_u64(void)301 _readgsbase_u64(void)
302 {
303   return __builtin_ia32_rdgsbase64();
304 }
305 
306 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_writefsbase_u32(unsigned int __V)307 _writefsbase_u32(unsigned int __V)
308 {
309   __builtin_ia32_wrfsbase32(__V);
310 }
311 
312 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_writefsbase_u64(unsigned long long __V)313 _writefsbase_u64(unsigned long long __V)
314 {
315   __builtin_ia32_wrfsbase64(__V);
316 }
317 
318 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_writegsbase_u32(unsigned int __V)319 _writegsbase_u32(unsigned int __V)
320 {
321   __builtin_ia32_wrgsbase32(__V);
322 }
323 
324 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_writegsbase_u64(unsigned long long __V)325 _writegsbase_u64(unsigned long long __V)
326 {
327   __builtin_ia32_wrgsbase64(__V);
328 }
329 
330 #endif
331 #endif /* __FSGSBASE__ */
332 
333 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
334     defined(__MOVBE__)
335 
336 /* The structs used below are to force the load/store to be unaligned. This
337  * is accomplished with the __packed__ attribute. The __may_alias__ prevents
338  * tbaa metadata from being generated based on the struct and the type of the
339  * field inside of it.
340  */
341 
342 static __inline__ short __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
_loadbe_i16(void const * __P)343 _loadbe_i16(void const * __P) {
344   struct __loadu_i16 {
345     short __v;
346   } __attribute__((__packed__, __may_alias__));
347   return __builtin_bswap16(((const struct __loadu_i16*)__P)->__v);
348 }
349 
350 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
_storebe_i16(void * __P,short __D)351 _storebe_i16(void * __P, short __D) {
352   struct __storeu_i16 {
353     short __v;
354   } __attribute__((__packed__, __may_alias__));
355   ((struct __storeu_i16*)__P)->__v = __builtin_bswap16(__D);
356 }
357 
358 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
_loadbe_i32(void const * __P)359 _loadbe_i32(void const * __P) {
360   struct __loadu_i32 {
361     int __v;
362   } __attribute__((__packed__, __may_alias__));
363   return __builtin_bswap32(((const struct __loadu_i32*)__P)->__v);
364 }
365 
366 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
_storebe_i32(void * __P,int __D)367 _storebe_i32(void * __P, int __D) {
368   struct __storeu_i32 {
369     int __v;
370   } __attribute__((__packed__, __may_alias__));
371   ((struct __storeu_i32*)__P)->__v = __builtin_bswap32(__D);
372 }
373 
374 #ifdef __x86_64__
375 static __inline__ long long __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
_loadbe_i64(void const * __P)376 _loadbe_i64(void const * __P) {
377   struct __loadu_i64 {
378     long long __v;
379   } __attribute__((__packed__, __may_alias__));
380   return __builtin_bswap64(((const struct __loadu_i64*)__P)->__v);
381 }
382 
383 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
_storebe_i64(void * __P,long long __D)384 _storebe_i64(void * __P, long long __D) {
385   struct __storeu_i64 {
386     long long __v;
387   } __attribute__((__packed__, __may_alias__));
388   ((struct __storeu_i64*)__P)->__v = __builtin_bswap64(__D);
389 }
390 #endif
391 #endif /* __MOVBE */
392 
393 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
394     defined(__RTM__)
395 #include <rtmintrin.h>
396 #include <xtestintrin.h>
397 #endif
398 
399 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
400     defined(__SHA__)
401 #include <shaintrin.h>
402 #endif
403 
404 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
405     defined(__FXSR__)
406 #include <fxsrintrin.h>
407 #endif
408 
409 /* No feature check desired due to internal MSC_VER checks */
410 #include <xsaveintrin.h>
411 
412 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
413     defined(__XSAVEOPT__)
414 #include <xsaveoptintrin.h>
415 #endif
416 
417 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
418     defined(__XSAVEC__)
419 #include <xsavecintrin.h>
420 #endif
421 
422 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
423     defined(__XSAVES__)
424 #include <xsavesintrin.h>
425 #endif
426 
427 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
428     defined(__SHSTK__)
429 #include <cetintrin.h>
430 #endif
431 
432 /* Some intrinsics inside adxintrin.h are available only on processors with ADX,
433  * whereas others are also available at all times. */
434 #include <adxintrin.h>
435 
436 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
437     defined(__RDSEED__)
438 #include <rdseedintrin.h>
439 #endif
440 
441 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
442     defined(__WBNOINVD__)
443 #include <wbnoinvdintrin.h>
444 #endif
445 
446 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
447     defined(__CLDEMOTE__)
448 #include <cldemoteintrin.h>
449 #endif
450 
451 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
452     defined(__WAITPKG__)
453 #include <waitpkgintrin.h>
454 #endif
455 
456 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
457     defined(__MOVDIRI__) || defined(__MOVDIR64B__)
458 #include <movdirintrin.h>
459 #endif
460 
461 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
462     defined(__PCONFIG__)
463 #include <pconfigintrin.h>
464 #endif
465 
466 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
467     defined(__SGX__)
468 #include <sgxintrin.h>
469 #endif
470 
471 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
472     defined(__PTWRITE__)
473 #include <ptwriteintrin.h>
474 #endif
475 
476 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
477     defined(__INVPCID__)
478 #include <invpcidintrin.h>
479 #endif
480 
481 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
482     defined(__KL__) || defined(__WIDEKL__)
483 #include <keylockerintrin.h>
484 #endif
485 
486 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
487     defined(__AMXTILE__) || defined(__AMXINT8__) || defined(__AMXBF16__)
488 #include <amxintrin.h>
489 #endif
490 
491 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
492     defined(__AVX512VP2INTERSECT__)
493 #include <avx512vp2intersectintrin.h>
494 #endif
495 
496 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
497     (defined(__AVX512VL__) && defined(__AVX512VP2INTERSECT__))
498 #include <avx512vlvp2intersectintrin.h>
499 #endif
500 
501 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
502     defined(__ENQCMD__)
503 #include <enqcmdintrin.h>
504 #endif
505 
506 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
507     defined(__SERIALIZE__)
508 #include <serializeintrin.h>
509 #endif
510 
511 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
512     defined(__TSXLDTRK__)
513 #include <tsxldtrkintrin.h>
514 #endif
515 
516 #if defined(_MSC_VER) && __has_extension(gnu_asm)
517 /* Define the default attributes for these intrinsics */
518 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
519 #ifdef __cplusplus
520 extern "C" {
521 #endif
522 /*----------------------------------------------------------------------------*\
523 |* Interlocked Exchange HLE
524 \*----------------------------------------------------------------------------*/
525 #if defined(__i386__) || defined(__x86_64__)
526 static __inline__ long __DEFAULT_FN_ATTRS
_InterlockedExchange_HLEAcquire(long volatile * _Target,long _Value)527 _InterlockedExchange_HLEAcquire(long volatile *_Target, long _Value) {
528   __asm__ __volatile__(".byte 0xf2 ; lock ; xchg %0, %1"
529                        : "+r" (_Value), "+m" (*_Target) :: "memory");
530   return _Value;
531 }
532 static __inline__ long __DEFAULT_FN_ATTRS
_InterlockedExchange_HLERelease(long volatile * _Target,long _Value)533 _InterlockedExchange_HLERelease(long volatile *_Target, long _Value) {
534   __asm__ __volatile__(".byte 0xf3 ; lock ; xchg %0, %1"
535                        : "+r" (_Value), "+m" (*_Target) :: "memory");
536   return _Value;
537 }
538 #endif
539 #if defined(__x86_64__)
540 static __inline__ __int64 __DEFAULT_FN_ATTRS
_InterlockedExchange64_HLEAcquire(__int64 volatile * _Target,__int64 _Value)541 _InterlockedExchange64_HLEAcquire(__int64 volatile *_Target, __int64 _Value) {
542   __asm__ __volatile__(".byte 0xf2 ; lock ; xchg %0, %1"
543                        : "+r" (_Value), "+m" (*_Target) :: "memory");
544   return _Value;
545 }
546 static __inline__ __int64 __DEFAULT_FN_ATTRS
_InterlockedExchange64_HLERelease(__int64 volatile * _Target,__int64 _Value)547 _InterlockedExchange64_HLERelease(__int64 volatile *_Target, __int64 _Value) {
548   __asm__ __volatile__(".byte 0xf3 ; lock ; xchg %0, %1"
549                        : "+r" (_Value), "+m" (*_Target) :: "memory");
550   return _Value;
551 }
552 #endif
553 /*----------------------------------------------------------------------------*\
554 |* Interlocked Compare Exchange HLE
555 \*----------------------------------------------------------------------------*/
556 #if defined(__i386__) || defined(__x86_64__)
557 static __inline__ long __DEFAULT_FN_ATTRS
_InterlockedCompareExchange_HLEAcquire(long volatile * _Destination,long _Exchange,long _Comparand)558 _InterlockedCompareExchange_HLEAcquire(long volatile *_Destination,
559                               long _Exchange, long _Comparand) {
560   __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg %2, %1"
561                        : "+a" (_Comparand), "+m" (*_Destination)
562                        : "r" (_Exchange) : "memory");
563   return _Comparand;
564 }
565 static __inline__ long __DEFAULT_FN_ATTRS
_InterlockedCompareExchange_HLERelease(long volatile * _Destination,long _Exchange,long _Comparand)566 _InterlockedCompareExchange_HLERelease(long volatile *_Destination,
567                               long _Exchange, long _Comparand) {
568   __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg %2, %1"
569                        : "+a" (_Comparand), "+m" (*_Destination)
570                        : "r" (_Exchange) : "memory");
571   return _Comparand;
572 }
573 #endif
574 #if defined(__x86_64__)
575 static __inline__ __int64 __DEFAULT_FN_ATTRS
_InterlockedCompareExchange64_HLEAcquire(__int64 volatile * _Destination,__int64 _Exchange,__int64 _Comparand)576 _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *_Destination,
577                               __int64 _Exchange, __int64 _Comparand) {
578   __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg %2, %1"
579                        : "+a" (_Comparand), "+m" (*_Destination)
580                        : "r" (_Exchange) : "memory");
581   return _Comparand;
582 }
583 static __inline__ __int64 __DEFAULT_FN_ATTRS
_InterlockedCompareExchange64_HLERelease(__int64 volatile * _Destination,__int64 _Exchange,__int64 _Comparand)584 _InterlockedCompareExchange64_HLERelease(__int64 volatile *_Destination,
585                               __int64 _Exchange, __int64 _Comparand) {
586   __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg %2, %1"
587                        : "+a" (_Comparand), "+m" (*_Destination)
588                        : "r" (_Exchange) : "memory");
589   return _Comparand;
590 }
591 #endif
592 #ifdef __cplusplus
593 }
594 #endif
595 
596 #undef __DEFAULT_FN_ATTRS
597 
598 #endif /* defined(_MSC_VER) && __has_extension(gnu_asm) */
599 
600 #endif /* __IMMINTRIN_H */
601