1; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs < %s | FileCheck %s
2; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-REG
3
4
5; Point of CHECK-REG is to make sure UNPREDICTABLE instructions aren't created
6; (i.e. reusing a register for status & data in store exclusive).
7; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], w[[NEW]], [x{{[0-9]+}}]
8; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], x[[NEW]], [x{{[0-9]+}}]
9
10@var8 = global i8 0
11@var16 = global i16 0
12@var32 = global i32 0
13@var64 = global i64 0
14
15define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
16; CHECK-LABEL: test_atomic_load_add_i8:
17   %old = atomicrmw add i8* @var8, i8 %offset seq_cst
18; CHECK-NOT: dmb
19; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
20; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
21
22; CHECK: .LBB{{[0-9]+}}_1:
23; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
24  ; w0 below is a reasonable guess but could change: it certainly comes into the
25  ;  function there.
26; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
27; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
28; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
29; CHECK-NOT: dmb
30
31; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
32   ret i8 %old
33}
34
35define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
36; CHECK-LABEL: test_atomic_load_add_i16:
37   %old = atomicrmw add i16* @var16, i16 %offset acquire
38; CHECK-NOT: dmb
39; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
40; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
41
42; CHECK: .LBB{{[0-9]+}}_1:
43; ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
44  ; w0 below is a reasonable guess but could change: it certainly comes into the
45  ;  function there.
46; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
47; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
48; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
49; CHECK-NOT: dmb
50
51; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
52   ret i16 %old
53}
54
55define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
56; CHECK-LABEL: test_atomic_load_add_i32:
57   %old = atomicrmw add i32* @var32, i32 %offset release
58; CHECK-NOT: dmb
59; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
60; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
61
62; CHECK: .LBB{{[0-9]+}}_1:
63; ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
64  ; w0 below is a reasonable guess but could change: it certainly comes into the
65  ;  function there.
66; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
67; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
68; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
69; CHECK-NOT: dmb
70
71; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
72   ret i32 %old
73}
74
75define i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
76; CHECK-LABEL: test_atomic_load_add_i64:
77   %old = atomicrmw add i64* @var64, i64 %offset monotonic
78; CHECK-NOT: dmb
79; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
80; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
81
82; CHECK: .LBB{{[0-9]+}}_1:
83; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
84  ; x0 below is a reasonable guess but could change: it certainly comes into the
85  ; function there.
86; CHECK-NEXT: add [[NEW:x[0-9]+]], x[[OLD]], x0
87; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
88; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
89; CHECK-NOT: dmb
90
91; CHECK: mov x0, x[[OLD]]
92   ret i64 %old
93}
94
95define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
96; CHECK-LABEL: test_atomic_load_sub_i8:
97   %old = atomicrmw sub i8* @var8, i8 %offset monotonic
98; CHECK-NOT: dmb
99; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
100; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
101
102; CHECK: .LBB{{[0-9]+}}_1:
103; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
104  ; w0 below is a reasonable guess but could change: it certainly comes into the
105  ;  function there.
106; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
107; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
108; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
109; CHECK-NOT: dmb
110
111; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
112   ret i8 %old
113}
114
115define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
116; CHECK-LABEL: test_atomic_load_sub_i16:
117   %old = atomicrmw sub i16* @var16, i16 %offset release
118; CHECK-NOT: dmb
119; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
120; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
121
122; CHECK: .LBB{{[0-9]+}}_1:
123; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
124  ; w0 below is a reasonable guess but could change: it certainly comes into the
125  ;  function there.
126; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
127; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
128; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
129; CHECK-NOT: dmb
130
131; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
132   ret i16 %old
133}
134
135define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
136; CHECK-LABEL: test_atomic_load_sub_i32:
137   %old = atomicrmw sub i32* @var32, i32 %offset acquire
138; CHECK-NOT: dmb
139; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
140; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
141
142; CHECK: .LBB{{[0-9]+}}_1:
143; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
144  ; w0 below is a reasonable guess but could change: it certainly comes into the
145  ;  function there.
146; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
147; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
148; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
149; CHECK-NOT: dmb
150
151; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
152   ret i32 %old
153}
154
155define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
156; CHECK-LABEL: test_atomic_load_sub_i64:
157   %old = atomicrmw sub i64* @var64, i64 %offset seq_cst
158; CHECK-NOT: dmb
159; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
160; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
161
162; CHECK: .LBB{{[0-9]+}}_1:
163; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
164  ; x0 below is a reasonable guess but could change: it certainly comes into the
165  ; function there.
166; CHECK-NEXT: sub [[NEW:x[0-9]+]], x[[OLD]], x0
167; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
168; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
169; CHECK-NOT: dmb
170
171; CHECK: mov x0, x[[OLD]]
172   ret i64 %old
173}
174
175define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
176; CHECK-LABEL: test_atomic_load_and_i8:
177   %old = atomicrmw and i8* @var8, i8 %offset release
178; CHECK-NOT: dmb
179; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
180; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
181
182; CHECK: .LBB{{[0-9]+}}_1:
183; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
184  ; w0 below is a reasonable guess but could change: it certainly comes into the
185  ;  function there.
186; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
187; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
188; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
189; CHECK-NOT: dmb
190
191; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
192   ret i8 %old
193}
194
195define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
196; CHECK-LABEL: test_atomic_load_and_i16:
197   %old = atomicrmw and i16* @var16, i16 %offset monotonic
198; CHECK-NOT: dmb
199; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
200; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
201
202; CHECK: .LBB{{[0-9]+}}_1:
203; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
204  ; w0 below is a reasonable guess but could change: it certainly comes into the
205  ;  function there.
206; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
207; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
208; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
209; CHECK-NOT: dmb
210
211; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
212   ret i16 %old
213}
214
215define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
216; CHECK-LABEL: test_atomic_load_and_i32:
217   %old = atomicrmw and i32* @var32, i32 %offset seq_cst
218; CHECK-NOT: dmb
219; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
220; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
221
222; CHECK: .LBB{{[0-9]+}}_1:
223; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
224  ; w0 below is a reasonable guess but could change: it certainly comes into the
225  ;  function there.
226; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
227; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
228; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
229; CHECK-NOT: dmb
230
231; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
232   ret i32 %old
233}
234
235define i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
236; CHECK-LABEL: test_atomic_load_and_i64:
237   %old = atomicrmw and i64* @var64, i64 %offset acquire
238; CHECK-NOT: dmb
239; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
240; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
241
242; CHECK: .LBB{{[0-9]+}}_1:
243; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
244  ; x0 below is a reasonable guess but could change: it certainly comes into the
245  ; function there.
246; CHECK-NEXT: and [[NEW:x[0-9]+]], x[[OLD]], x0
247; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
248; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
249; CHECK-NOT: dmb
250
251; CHECK: mov x0, x[[OLD]]
252   ret i64 %old
253}
254
255define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
256; CHECK-LABEL: test_atomic_load_or_i8:
257   %old = atomicrmw or i8* @var8, i8 %offset seq_cst
258; CHECK-NOT: dmb
259; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
260; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
261
262; CHECK: .LBB{{[0-9]+}}_1:
263; ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
264  ; w0 below is a reasonable guess but could change: it certainly comes into the
265  ;  function there.
266; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
267; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
268; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
269; CHECK-NOT: dmb
270
271; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
272   ret i8 %old
273}
274
275define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
276; CHECK-LABEL: test_atomic_load_or_i16:
277   %old = atomicrmw or i16* @var16, i16 %offset monotonic
278; CHECK-NOT: dmb
279; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
280; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
281
282; CHECK: .LBB{{[0-9]+}}_1:
283; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
284  ; w0 below is a reasonable guess but could change: it certainly comes into the
285  ;  function there.
286; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
287; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
288; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
289; CHECK-NOT: dmb
290
291; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
292   ret i16 %old
293}
294
295define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
296; CHECK-LABEL: test_atomic_load_or_i32:
297   %old = atomicrmw or i32* @var32, i32 %offset acquire
298; CHECK-NOT: dmb
299; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
300; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
301
302; CHECK: .LBB{{[0-9]+}}_1:
303; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
304  ; w0 below is a reasonable guess but could change: it certainly comes into the
305  ;  function there.
306; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
307; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
308; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
309; CHECK-NOT: dmb
310
311; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
312   ret i32 %old
313}
314
315define i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
316; CHECK-LABEL: test_atomic_load_or_i64:
317   %old = atomicrmw or i64* @var64, i64 %offset release
318; CHECK-NOT: dmb
319; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
320; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
321
322; CHECK: .LBB{{[0-9]+}}_1:
323; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
324  ; x0 below is a reasonable guess but could change: it certainly comes into the
325  ; function there.
326; CHECK-NEXT: orr [[NEW:x[0-9]+]], x[[OLD]], x0
327; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
328; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
329; CHECK-NOT: dmb
330
331; CHECK: mov x0, x[[OLD]]
332   ret i64 %old
333}
334
335define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
336; CHECK-LABEL: test_atomic_load_xor_i8:
337   %old = atomicrmw xor i8* @var8, i8 %offset acquire
338; CHECK-NOT: dmb
339; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
340; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
341
342; CHECK: .LBB{{[0-9]+}}_1:
343; ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
344  ; w0 below is a reasonable guess but could change: it certainly comes into the
345  ;  function there.
346; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
347; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
348; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
349; CHECK-NOT: dmb
350
351; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
352   ret i8 %old
353}
354
355define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
356; CHECK-LABEL: test_atomic_load_xor_i16:
357   %old = atomicrmw xor i16* @var16, i16 %offset release
358; CHECK-NOT: dmb
359; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
360; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
361
362; CHECK: .LBB{{[0-9]+}}_1:
363; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
364  ; w0 below is a reasonable guess but could change: it certainly comes into the
365  ;  function there.
366; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
367; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
368; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
369; CHECK-NOT: dmb
370
371; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
372   ret i16 %old
373}
374
375define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
376; CHECK-LABEL: test_atomic_load_xor_i32:
377   %old = atomicrmw xor i32* @var32, i32 %offset seq_cst
378; CHECK-NOT: dmb
379; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
380; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
381
382; CHECK: .LBB{{[0-9]+}}_1:
383; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
384  ; w0 below is a reasonable guess but could change: it certainly comes into the
385  ;  function there.
386; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
387; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
388; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
389; CHECK-NOT: dmb
390
391; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
392   ret i32 %old
393}
394
395define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
396; CHECK-LABEL: test_atomic_load_xor_i64:
397   %old = atomicrmw xor i64* @var64, i64 %offset monotonic
398; CHECK-NOT: dmb
399; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
400; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
401
402; CHECK: .LBB{{[0-9]+}}_1:
403; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
404  ; x0 below is a reasonable guess but could change: it certainly comes into the
405  ; function there.
406; CHECK-NEXT: eor [[NEW:x[0-9]+]], x[[OLD]], x0
407; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
408; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
409; CHECK-NOT: dmb
410
411; CHECK: mov x0, x[[OLD]]
412   ret i64 %old
413}
414
415define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
416; CHECK-LABEL: test_atomic_load_xchg_i8:
417   %old = atomicrmw xchg i8* @var8, i8 %offset monotonic
418; CHECK-NOT: dmb
419; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
420; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
421
422; CHECK: .LBB{{[0-9]+}}_1:
423; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
424  ; w0 below is a reasonable guess but could change: it certainly comes into the
425  ; function there.
426; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
427; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
428; CHECK-NOT: dmb
429
430; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
431   ret i8 %old
432}
433
434define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
435; CHECK-LABEL: test_atomic_load_xchg_i16:
436   %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst
437; CHECK-NOT: dmb
438; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
439; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
440
441; CHECK: .LBB{{[0-9]+}}_1:
442; ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
443  ; w0 below is a reasonable guess but could change: it certainly comes into the
444  ; function there.
445; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
446; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
447; CHECK-NOT: dmb
448
449; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
450   ret i16 %old
451}
452
453define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
454; CHECK-LABEL: test_atomic_load_xchg_i32:
455; CHECK: mov {{[xw]}}8, w[[OLD:[0-9]+]]
456   %old = atomicrmw xchg i32* @var32, i32 %offset release
457; CHECK-NOT: dmb
458; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
459; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
460
461; CHECK: .LBB{{[0-9]+}}_1:
462; ; CHECK: ldxr {{[xw]}}[[OLD]], [x[[ADDR]]]
463  ; w0 below is a reasonable guess but could change: it certainly comes into the
464  ;  function there.
465; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], w8, [x[[ADDR]]]
466; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
467; CHECK-NOT: dmb
468   ret i32 %old
469}
470
471define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
472; CHECK-LABEL: test_atomic_load_xchg_i64:
473   %old = atomicrmw xchg i64* @var64, i64 %offset acquire
474; CHECK-NOT: dmb
475; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
476; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
477
478; CHECK: .LBB{{[0-9]+}}_1:
479; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
480  ; x0 below is a reasonable guess but could change: it certainly comes into the
481  ; function there.
482; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], x0, [x[[ADDR]]]
483; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
484; CHECK-NOT: dmb
485
486; CHECK: mov x0, x[[OLD]]
487   ret i64 %old
488}
489
490
491define i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
492; CHECK-LABEL: test_atomic_load_min_i8:
493   %old = atomicrmw min i8* @var8, i8 %offset acquire
494; CHECK-NOT: dmb
495; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
496; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
497
498; CHECK: .LBB{{[0-9]+}}_1:
499; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
500  ; w0 below is a reasonable guess but could change: it certainly comes into the
501  ;  function there.
502
503; CHECK-NEXT: sxtb w[[OLD_EXT:[0-9]+]], w[[OLD]]
504; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxtb
505; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
506
507; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
508; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
509; CHECK-NOT: dmb
510
511; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
512   ret i8 %old
513}
514
515define i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
516; CHECK-LABEL: test_atomic_load_min_i16:
517   %old = atomicrmw min i16* @var16, i16 %offset release
518; CHECK-NOT: dmb
519; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
520; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
521
522; CHECK: .LBB{{[0-9]+}}_1:
523; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
524  ; w0 below is a reasonable guess but could change: it certainly comes into the
525  ;  function there.
526
527; CHECK-NEXT: sxth w[[OLD_EXT:[0-9]+]], w[[OLD]]
528; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxth
529; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
530
531
532; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
533; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
534; CHECK-NOT: dmb
535
536; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
537   ret i16 %old
538}
539
540define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
541; CHECK-LABEL: test_atomic_load_min_i32:
542   %old = atomicrmw min i32* @var32, i32 %offset monotonic
543; CHECK-NOT: dmb
544; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
545; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
546
547; CHECK: .LBB{{[0-9]+}}_1:
548; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
549  ; w0 below is a reasonable guess but could change: it certainly comes into the
550  ;  function there.
551
552; CHECK-NEXT: cmp w[[OLD]], w0
553; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
554
555
556; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
557; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
558; CHECK-NOT: dmb
559
560; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
561   ret i32 %old
562}
563
564define i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
565; CHECK-LABEL: test_atomic_load_min_i64:
566   %old = atomicrmw min i64* @var64, i64 %offset seq_cst
567; CHECK-NOT: dmb
568; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
569; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
570
571; CHECK: .LBB{{[0-9]+}}_1:
572; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
573  ; x0 below is a reasonable guess but could change: it certainly comes into the
574  ; function there.
575
576; CHECK-NEXT: cmp x[[OLD]], x0
577; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, le
578
579
580; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
581; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
582; CHECK-NOT: dmb
583
584; CHECK: mov x0, x[[OLD]]
585   ret i64 %old
586}
587
588define i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
589; CHECK-LABEL: test_atomic_load_max_i8:
590   %old = atomicrmw max i8* @var8, i8 %offset seq_cst
591; CHECK-NOT: dmb
592; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
593; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
594
595; CHECK: .LBB{{[0-9]+}}_1:
596; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
597  ; w0 below is a reasonable guess but could change: it certainly comes into the
598  ;  function there.
599
600; CHECK-NEXT: sxtb w[[OLD_EXT:[0-9]+]], w[[OLD]]
601; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxtb
602; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
603
604
605; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
606; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
607; CHECK-NOT: dmb
608
609; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
610   ret i8 %old
611}
612
613define i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
614; CHECK-LABEL: test_atomic_load_max_i16:
615   %old = atomicrmw max i16* @var16, i16 %offset acquire
616; CHECK-NOT: dmb
617; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
618; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
619
620; CHECK: .LBB{{[0-9]+}}_1:
621; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
622  ; w0 below is a reasonable guess but could change: it certainly comes into the
623  ;  function there.
624
625; CHECK-NEXT: sxth w[[OLD_EXT:[0-9]+]], w[[OLD]]
626; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxth
627; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
628
629
630; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
631; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
632; CHECK-NOT: dmb
633
634; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
635   ret i16 %old
636}
637
638define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
639; CHECK-LABEL: test_atomic_load_max_i32:
640   %old = atomicrmw max i32* @var32, i32 %offset release
641; CHECK-NOT: dmb
642; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
643; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
644
645; CHECK: .LBB{{[0-9]+}}_1:
646; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
647  ; w0 below is a reasonable guess but could change: it certainly comes into the
648  ;  function there.
649
650; CHECK-NEXT: cmp w[[OLD]], w0
651; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
652
653
654; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
655; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
656; CHECK-NOT: dmb
657
658; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
659   ret i32 %old
660}
661
662define i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
663; CHECK-LABEL: test_atomic_load_max_i64:
664   %old = atomicrmw max i64* @var64, i64 %offset monotonic
665; CHECK-NOT: dmb
666; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
667; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
668
669; CHECK: .LBB{{[0-9]+}}_1:
670; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
671  ; x0 below is a reasonable guess but could change: it certainly comes into the
672  ; function there.
673
674; CHECK-NEXT: cmp x[[OLD]], x0
675; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, gt
676
677
678; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
679; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
680; CHECK-NOT: dmb
681
682; CHECK: mov x0, x[[OLD]]
683   ret i64 %old
684}
685
686define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
687; CHECK-LABEL: test_atomic_load_umin_i8:
688   %old = atomicrmw umin i8* @var8, i8 %offset monotonic
689; CHECK-NOT: dmb
690; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
691; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
692
693; CHECK: .LBB{{[0-9]+}}_1:
694; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
695  ; w0 below is a reasonable guess but could change: it certainly comes into the
696  ;  function there.
697
698; CHECK-NEXT: cmp w[[OLD]], w0, uxtb
699; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
700
701
702; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
703; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
704; CHECK-NOT: dmb
705
706; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
707   ret i8 %old
708}
709
710define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
711; CHECK-LABEL: test_atomic_load_umin_i16:
712   %old = atomicrmw umin i16* @var16, i16 %offset acquire
713; CHECK-NOT: dmb
714; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
715; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
716
717; CHECK: .LBB{{[0-9]+}}_1:
718; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
719  ; w0 below is a reasonable guess but could change: it certainly comes into the
720  ;  function there.
721
722; CHECK-NEXT: cmp w[[OLD]], w0, uxth
723; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
724
725
726; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
727; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
728; CHECK-NOT: dmb
729
730; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
731   ret i16 %old
732}
733
734define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
735; CHECK-LABEL: test_atomic_load_umin_i32:
736   %old = atomicrmw umin i32* @var32, i32 %offset seq_cst
737; CHECK-NOT: dmb
738; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
739; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
740
741; CHECK: .LBB{{[0-9]+}}_1:
742; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
743  ; w0 below is a reasonable guess but could change: it certainly comes into the
744  ;  function there.
745
746; CHECK-NEXT: cmp w[[OLD]], w0
747; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
748
749
750; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
751; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
752; CHECK-NOT: dmb
753
754; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
755   ret i32 %old
756}
757
758define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
759; CHECK-LABEL: test_atomic_load_umin_i64:
760   %old = atomicrmw umin i64* @var64, i64 %offset acq_rel
761; CHECK-NOT: dmb
762; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
763; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
764
765; CHECK: .LBB{{[0-9]+}}_1:
766; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
767  ; x0 below is a reasonable guess but could change: it certainly comes into the
768  ; function there.
769
770; CHECK-NEXT: cmp x[[OLD]], x0
771; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, ls
772
773
774; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
775; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
776; CHECK-NOT: dmb
777
778; CHECK: mov x0, x[[OLD]]
779   ret i64 %old
780}
781
782define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
783; CHECK-LABEL: test_atomic_load_umax_i8:
784   %old = atomicrmw umax i8* @var8, i8 %offset acq_rel
785; CHECK-NOT: dmb
786; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
787; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
788
789; CHECK: .LBB{{[0-9]+}}_1:
790; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
791  ; w0 below is a reasonable guess but could change: it certainly comes into the
792  ;  function there.
793
794; CHECK-NEXT: cmp w[[OLD]], w0, uxtb
795; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
796
797
798; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
799; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
800; CHECK-NOT: dmb
801
802; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
803   ret i8 %old
804}
805
806define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
807; CHECK-LABEL: test_atomic_load_umax_i16:
808   %old = atomicrmw umax i16* @var16, i16 %offset monotonic
809; CHECK-NOT: dmb
810; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
811; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
812
813; CHECK: .LBB{{[0-9]+}}_1:
814; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
815  ; w0 below is a reasonable guess but could change: it certainly comes into the
816  ;  function there.
817
818; CHECK-NEXT: cmp w[[OLD]], w0, uxth
819; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
820
821
822; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
823; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
824; CHECK-NOT: dmb
825
826; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
827   ret i16 %old
828}
829
830define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
831; CHECK-LABEL: test_atomic_load_umax_i32:
832   %old = atomicrmw umax i32* @var32, i32 %offset seq_cst
833; CHECK-NOT: dmb
834; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
835; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
836
837; CHECK: .LBB{{[0-9]+}}_1:
838; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
839  ; w0 below is a reasonable guess but could change: it certainly comes into the
840  ;  function there.
841
842; CHECK-NEXT: cmp w[[OLD]], w0
843; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
844
845
846; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
847; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
848; CHECK-NOT: dmb
849
850; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
851   ret i32 %old
852}
853
854define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
855; CHECK-LABEL: test_atomic_load_umax_i64:
856   %old = atomicrmw umax i64* @var64, i64 %offset release
857; CHECK-NOT: dmb
858; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
859; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
860
861; CHECK: .LBB{{[0-9]+}}_1:
862; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
863  ; x0 below is a reasonable guess but could change: it certainly comes into the
864  ; function there.
865
866; CHECK-NEXT: cmp x[[OLD]], x0
867; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, hi
868
869
870; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
871; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
872; CHECK-NOT: dmb
873
874; CHECK: mov x0, x[[OLD]]
875   ret i64 %old
876}
877
878define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
879; CHECK-LABEL: test_atomic_cmpxchg_i8:
880   %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
881   %old = extractvalue { i8, i1 } %pair, 0
882
883; CHECK-NOT: dmb
884; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
885; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
886
887; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
888; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
889  ; w0 below is a reasonable guess but could change: it certainly comes into the
890  ;  function there.
891; CHECK-NEXT: cmp w[[OLD]], w0
892; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
893; CHECK: stxrb [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
894; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
895; CHECK: [[GET_OUT]]:
896; CHECK: clrex
897; CHECK-NOT: dmb
898
899; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
900   ret i8 %old
901}
902
903define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
904; CHECK-LABEL: test_atomic_cmpxchg_i16:
905   %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst
906   %old = extractvalue { i16, i1 } %pair, 0
907
908; CHECK-NOT: dmb
909; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
910; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
911
912; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
913; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
914  ; w0 below is a reasonable guess but could change: it certainly comes into the
915  ;  function there.
916; CHECK-NEXT: cmp w[[OLD]], w0
917; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
918; CHECK: stlxrh [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
919; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
920; CHECK: [[GET_OUT]]:
921; CHECK: clrex
922; CHECK-NOT: dmb
923
924; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
925   ret i16 %old
926}
927
928define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
929; CHECK-LABEL: test_atomic_cmpxchg_i32:
930   %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic
931   %old = extractvalue { i32, i1 } %pair, 0
932
933; CHECK: mov {{[xw]}}[[WANTED:[0-9]+]], {{[xw]}}0
934
935; CHECK-NOT: dmb
936; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
937; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
938
939; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
940; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
941; CHECK-NEXT: cmp w[[OLD]], w[[WANTED]]
942; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
943; CHECK: stlxr [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
944; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
945; CHECK: [[GET_OUT]]:
946; CHECK: clrex
947; CHECK-NOT: dmb
948   ret i32 %old
949}
950
951define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
952; CHECK-LABEL: test_atomic_cmpxchg_i64:
953   %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic
954   %old = extractvalue { i64, i1 } %pair, 0
955
956; CHECK-NOT: dmb
957; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
958; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
959
960; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
961; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
962  ; w0 below is a reasonable guess but could change: it certainly comes into the
963  ;  function there.
964; CHECK-NEXT: cmp x[[OLD]], x0
965; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
966  ; As above, w1 is a reasonable guess.
967; CHECK: stxr [[STATUS:w[0-9]+]], x1, [x[[ADDR]]]
968; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
969; CHECK: [[GET_OUT]]:
970; CHECK: clrex
971; CHECK-NOT: dmb
972
973; CHECK: str x[[OLD]],
974   store i64 %old, i64* @var64
975   ret void
976}
977
978define i8 @test_atomic_load_monotonic_i8() nounwind {
979; CHECK-LABEL: test_atomic_load_monotonic_i8:
980  %val = load atomic i8, i8* @var8 monotonic, align 1
981; CHECK-NOT: dmb
982; CHECK: adrp x[[HIADDR:[0-9]+]], var8
983; CHECK: ldrb w0, [x[[HIADDR]], {{#?}}:lo12:var8]
984; CHECK-NOT: dmb
985
986  ret i8 %val
987}
988
989define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
990; CHECK-LABEL: test_atomic_load_monotonic_regoff_i8:
991  %addr_int = add i64 %base, %off
992  %addr = inttoptr i64 %addr_int to i8*
993
994  %val = load atomic i8, i8* %addr monotonic, align 1
995; CHECK-NOT: dmb
996; CHECK: ldrb w0, [x0, x1]
997; CHECK-NOT: dmb
998
999  ret i8 %val
1000}
1001
1002define i8 @test_atomic_load_acquire_i8() nounwind {
1003; CHECK-LABEL: test_atomic_load_acquire_i8:
1004  %val = load atomic i8, i8* @var8 acquire, align 1
1005; CHECK-NOT: dmb
1006; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1007; CHECK-NOT: dmb
1008; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1009; CHECK-NOT: dmb
1010; CHECK: ldarb w0, [x[[ADDR]]]
1011; CHECK-NOT: dmb
1012  ret i8 %val
1013}
1014
1015define i8 @test_atomic_load_seq_cst_i8() nounwind {
1016; CHECK-LABEL: test_atomic_load_seq_cst_i8:
1017  %val = load atomic i8, i8* @var8 seq_cst, align 1
1018; CHECK-NOT: dmb
1019; CHECK: adrp [[HIADDR:x[0-9]+]], var8
1020; CHECK-NOT: dmb
1021; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
1022; CHECK-NOT: dmb
1023; CHECK: ldarb w0, [x[[ADDR]]]
1024; CHECK-NOT: dmb
1025  ret i8 %val
1026}
1027
1028define i16 @test_atomic_load_monotonic_i16() nounwind {
1029; CHECK-LABEL: test_atomic_load_monotonic_i16:
1030  %val = load atomic i16, i16* @var16 monotonic, align 2
1031; CHECK-NOT: dmb
1032; CHECK: adrp x[[HIADDR:[0-9]+]], var16
1033; CHECK-NOT: dmb
1034; CHECK: ldrh w0, [x[[HIADDR]], {{#?}}:lo12:var16]
1035; CHECK-NOT: dmb
1036
1037  ret i16 %val
1038}
1039
1040define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {
1041; CHECK-LABEL: test_atomic_load_monotonic_regoff_i32:
1042  %addr_int = add i64 %base, %off
1043  %addr = inttoptr i64 %addr_int to i32*
1044
1045  %val = load atomic i32, i32* %addr monotonic, align 4
1046; CHECK-NOT: dmb
1047; CHECK: ldr w0, [x0, x1]
1048; CHECK-NOT: dmb
1049
1050  ret i32 %val
1051}
1052
1053define i64 @test_atomic_load_seq_cst_i64() nounwind {
1054; CHECK-LABEL: test_atomic_load_seq_cst_i64:
1055  %val = load atomic i64, i64* @var64 seq_cst, align 8
1056; CHECK-NOT: dmb
1057; CHECK: adrp [[HIADDR:x[0-9]+]], var64
1058; CHECK-NOT: dmb
1059; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var64
1060; CHECK-NOT: dmb
1061; CHECK: ldar x0, [x[[ADDR]]]
1062; CHECK-NOT: dmb
1063  ret i64 %val
1064}
1065
1066define void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
1067; CHECK-LABEL: test_atomic_store_monotonic_i8:
1068  store atomic i8 %val, i8* @var8 monotonic, align 1
1069; CHECK: adrp x[[HIADDR:[0-9]+]], var8
1070; CHECK: strb w0, [x[[HIADDR]], {{#?}}:lo12:var8]
1071
1072  ret void
1073}
1074
1075define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {
1076; CHECK-LABEL: test_atomic_store_monotonic_regoff_i8:
1077
1078  %addr_int = add i64 %base, %off
1079  %addr = inttoptr i64 %addr_int to i8*
1080
1081  store atomic i8 %val, i8* %addr monotonic, align 1
1082; CHECK: strb w2, [x0, x1]
1083
1084  ret void
1085}
1086define void @test_atomic_store_release_i8(i8 %val) nounwind {
1087; CHECK-LABEL: test_atomic_store_release_i8:
1088  store atomic i8 %val, i8* @var8 release, align 1
1089; CHECK-NOT: dmb
1090; CHECK: adrp [[HIADDR:x[0-9]+]], var8
1091; CHECK-NOT: dmb
1092; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
1093; CHECK-NOT: dmb
1094; CHECK: stlrb w0, [x[[ADDR]]]
1095; CHECK-NOT: dmb
1096  ret void
1097}
1098
1099define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
1100; CHECK-LABEL: test_atomic_store_seq_cst_i8:
1101  store atomic i8 %val, i8* @var8 seq_cst, align 1
1102; CHECK-NOT: dmb
1103; CHECK: adrp [[HIADDR:x[0-9]+]], var8
1104; CHECK-NOT: dmb
1105; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
1106; CHECK-NOT: dmb
1107; CHECK: stlrb w0, [x[[ADDR]]]
1108; CHECK-NOT: dmb
1109
1110  ret void
1111}
1112
1113define void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
1114; CHECK-LABEL: test_atomic_store_monotonic_i16:
1115  store atomic i16 %val, i16* @var16 monotonic, align 2
1116; CHECK-NOT: dmb
1117; CHECK: adrp x[[HIADDR:[0-9]+]], var16
1118; CHECK-NOT: dmb
1119; CHECK: strh w0, [x[[HIADDR]], {{#?}}:lo12:var16]
1120; CHECK-NOT: dmb
1121  ret void
1122}
1123
1124define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {
1125; CHECK-LABEL: test_atomic_store_monotonic_regoff_i32:
1126
1127  %addr_int = add i64 %base, %off
1128  %addr = inttoptr i64 %addr_int to i32*
1129
1130  store atomic i32 %val, i32* %addr monotonic, align 4
1131; CHECK-NOT: dmb
1132; CHECK: str w2, [x0, x1]
1133; CHECK-NOT: dmb
1134
1135  ret void
1136}
1137
1138define void @test_atomic_store_release_i64(i64 %val) nounwind {
1139; CHECK-LABEL: test_atomic_store_release_i64:
1140  store atomic i64 %val, i64* @var64 release, align 8
1141; CHECK-NOT: dmb
1142; CHECK: adrp [[HIADDR:x[0-9]+]], var64
1143; CHECK-NOT: dmb
1144; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var64
1145; CHECK-NOT: dmb
1146; CHECK: stlr x0, [x[[ADDR]]]
1147; CHECK-NOT: dmb
1148  ret void
1149}
1150