1; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+lse < %s | FileCheck %s
2; RUN: llc -mtriple=aarch64_be-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+lse < %s | FileCheck %s
3; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+lse < %s | FileCheck %s --check-prefix=CHECK-REG
4; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mcpu=saphira < %s | FileCheck %s
5
6; Point of CHECK-REG is to make sure UNPREDICTABLE instructions aren't created
7; (i.e. reusing a register for status & data in store exclusive).
8; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], w[[NEW]], [x{{[0-9]+}}]
9; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], x[[NEW]], [x{{[0-9]+}}]
10
11@var8 = global i8 0
12@var16 = global i16 0
13@var32 = global i32 0
14@var64 = global i64 0
15@var128 = global i128 0
16
17define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
18; CHECK-LABEL: test_atomic_load_add_i8:
19   %old = atomicrmw add i8* @var8, i8 %offset seq_cst
20; CHECK-NOT: dmb
21; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
22; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
23
24; CHECK: ldaddalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
25; CHECK-NOT: dmb
26
27   ret i8 %old
28}
29
30define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
31; CHECK-LABEL: test_atomic_load_add_i16:
32   %old = atomicrmw add i16* @var16, i16 %offset seq_cst
33; CHECK-NOT: dmb
34; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
35; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
36
37; CHECK: ldaddalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
38; CHECK-NOT: dmb
39
40   ret i16 %old
41}
42
43define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
44; CHECK-LABEL: test_atomic_load_add_i32:
45   %old = atomicrmw add i32* @var32, i32 %offset seq_cst
46; CHECK-NOT: dmb
47; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
48; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
49
50; CHECK: ldaddal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
51; CHECK-NOT: dmb
52
53   ret i32 %old
54}
55
56define i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
57; CHECK-LABEL: test_atomic_load_add_i64:
58   %old = atomicrmw add i64* @var64, i64 %offset seq_cst
59; CHECK-NOT: dmb
60; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
61; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
62
63; CHECK: ldaddal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
64; CHECK-NOT: dmb
65
66   ret i64 %old
67}
68
69define void @test_atomic_load_add_i32_noret(i32 %offset) nounwind {
70; CHECK-LABEL: test_atomic_load_add_i32_noret:
71   atomicrmw add i32* @var32, i32 %offset seq_cst
72; CHECK-NOT: dmb
73; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
74; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
75
76; CHECK: ldaddal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
77; CHECK-NOT: dmb
78  ret void
79}
80
81define void @test_atomic_load_add_i64_noret(i64 %offset) nounwind {
82; CHECK-LABEL: test_atomic_load_add_i64_noret:
83   atomicrmw add i64* @var64, i64 %offset seq_cst
84; CHECK-NOT: dmb
85; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
86; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
87
88; CHECK: ldaddal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
89; CHECK-NOT: dmb
90  ret void
91}
92
93define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
94; CHECK-LABEL: test_atomic_load_or_i8:
95   %old = atomicrmw or i8* @var8, i8 %offset seq_cst
96; CHECK-NOT: dmb
97; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
98; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
99
100; CHECK: ldsetalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
101; CHECK-NOT: dmb
102
103   ret i8 %old
104}
105
106define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
107; CHECK-LABEL: test_atomic_load_or_i16:
108   %old = atomicrmw or i16* @var16, i16 %offset seq_cst
109; CHECK-NOT: dmb
110; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
111; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
112
113; CHECK: ldsetalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
114; CHECK-NOT: dmb
115
116   ret i16 %old
117}
118
119define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
120; CHECK-LABEL: test_atomic_load_or_i32:
121   %old = atomicrmw or i32* @var32, i32 %offset seq_cst
122; CHECK-NOT: dmb
123; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
124; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
125
126; CHECK: ldsetal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
127; CHECK-NOT: dmb
128
129   ret i32 %old
130}
131
132define i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
133; CHECK-LABEL: test_atomic_load_or_i64:
134   %old = atomicrmw or i64* @var64, i64 %offset seq_cst
135; CHECK-NOT: dmb
136; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
137; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
138
139; CHECK: ldsetal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
140; CHECK-NOT: dmb
141
142   ret i64 %old
143}
144
145define void @test_atomic_load_or_i32_noret(i32 %offset) nounwind {
146; CHECK-LABEL: test_atomic_load_or_i32_noret:
147   atomicrmw or i32* @var32, i32 %offset seq_cst
148; CHECK-NOT: dmb
149; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
150; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
151
152; CHECK: ldsetal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
153; CHECK-NOT: dmb
154  ret void
155}
156
157define void @test_atomic_load_or_i64_noret(i64 %offset) nounwind {
158; CHECK-LABEL: test_atomic_load_or_i64_noret:
159   atomicrmw or i64* @var64, i64 %offset seq_cst
160; CHECK-NOT: dmb
161; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
162; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
163
164; CHECK: ldsetal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
165; CHECK-NOT: dmb
166  ret void
167}
168
169define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
170; CHECK-LABEL: test_atomic_load_xor_i8:
171   %old = atomicrmw xor i8* @var8, i8 %offset seq_cst
172; CHECK-NOT: dmb
173; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
174; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
175
176; CHECK: ldeoralb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
177; CHECK-NOT: dmb
178
179   ret i8 %old
180}
181
182define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
183; CHECK-LABEL: test_atomic_load_xor_i16:
184   %old = atomicrmw xor i16* @var16, i16 %offset seq_cst
185; CHECK-NOT: dmb
186; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
187; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
188
189; CHECK: ldeoralh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
190; CHECK-NOT: dmb
191
192   ret i16 %old
193}
194
195define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
196; CHECK-LABEL: test_atomic_load_xor_i32:
197   %old = atomicrmw xor i32* @var32, i32 %offset seq_cst
198; CHECK-NOT: dmb
199; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
200; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
201
202; CHECK: ldeoral w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
203; CHECK-NOT: dmb
204
205   ret i32 %old
206}
207
208define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
209; CHECK-LABEL: test_atomic_load_xor_i64:
210   %old = atomicrmw xor i64* @var64, i64 %offset seq_cst
211; CHECK-NOT: dmb
212; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
213; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
214
215; CHECK: ldeoral x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
216; CHECK-NOT: dmb
217
218   ret i64 %old
219}
220
221define void @test_atomic_load_xor_i32_noret(i32 %offset) nounwind {
222; CHECK-LABEL: test_atomic_load_xor_i32_noret:
223   atomicrmw xor i32* @var32, i32 %offset seq_cst
224; CHECK-NOT: dmb
225; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
226; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
227
228; CHECK: ldeoral w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
229; CHECK-NOT: dmb
230  ret void
231}
232
233define void @test_atomic_load_xor_i64_noret(i64 %offset) nounwind {
234; CHECK-LABEL: test_atomic_load_xor_i64_noret:
235   atomicrmw xor i64* @var64, i64 %offset seq_cst
236; CHECK-NOT: dmb
237; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
238; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
239
240; CHECK: ldeoral x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
241; CHECK-NOT: dmb
242  ret void
243}
244
245define i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
246; CHECK-LABEL: test_atomic_load_min_i8:
247   %old = atomicrmw min i8* @var8, i8 %offset seq_cst
248; CHECK-NOT: dmb
249; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
250; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
251
252; CHECK: ldsminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
253; CHECK-NOT: dmb
254
255   ret i8 %old
256}
257
258define i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
259; CHECK-LABEL: test_atomic_load_min_i16:
260   %old = atomicrmw min i16* @var16, i16 %offset seq_cst
261; CHECK-NOT: dmb
262; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
263; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
264
265; CHECK: ldsminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
266; CHECK-NOT: dmb
267
268   ret i16 %old
269}
270
271define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
272; CHECK-LABEL: test_atomic_load_min_i32:
273   %old = atomicrmw min i32* @var32, i32 %offset seq_cst
274; CHECK-NOT: dmb
275; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
276; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
277
278; CHECK: ldsminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
279; CHECK-NOT: dmb
280
281   ret i32 %old
282}
283
284define i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
285; CHECK-LABEL: test_atomic_load_min_i64:
286   %old = atomicrmw min i64* @var64, i64 %offset seq_cst
287; CHECK-NOT: dmb
288; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
289; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
290
291; CHECK: ldsminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
292; CHECK-NOT: dmb
293
294   ret i64 %old
295}
296
297define void @test_atomic_load_min_i32_noret(i32 %offset) nounwind {
298; CHECK-LABEL: test_atomic_load_min_i32_noret:
299   atomicrmw min i32* @var32, i32 %offset seq_cst
300; CHECK-NOT: dmb
301; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
302; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
303
304; CHECK: ldsminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
305; CHECK-NOT: dmb
306  ret void
307}
308
309define void @test_atomic_load_min_i64_noret(i64 %offset) nounwind {
310; CHECK-LABEL: test_atomic_load_min_i64_noret:
311   atomicrmw min i64* @var64, i64 %offset seq_cst
312; CHECK-NOT: dmb
313; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
314; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
315
316; CHECK: ldsminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
317; CHECK-NOT: dmb
318  ret void
319}
320
321define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
322; CHECK-LABEL: test_atomic_load_umin_i8:
323   %old = atomicrmw umin i8* @var8, i8 %offset seq_cst
324; CHECK-NOT: dmb
325; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
326; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
327
328; CHECK: lduminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
329; CHECK-NOT: dmb
330
331   ret i8 %old
332}
333
334define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
335; CHECK-LABEL: test_atomic_load_umin_i16:
336   %old = atomicrmw umin i16* @var16, i16 %offset seq_cst
337; CHECK-NOT: dmb
338; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
339; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
340
341; CHECK: lduminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
342; CHECK-NOT: dmb
343
344   ret i16 %old
345}
346
347define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
348; CHECK-LABEL: test_atomic_load_umin_i32:
349   %old = atomicrmw umin i32* @var32, i32 %offset seq_cst
350; CHECK-NOT: dmb
351; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
352; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
353
354; CHECK: lduminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
355; CHECK-NOT: dmb
356
357   ret i32 %old
358}
359
360define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
361; CHECK-LABEL: test_atomic_load_umin_i64:
362   %old = atomicrmw umin i64* @var64, i64 %offset seq_cst
363; CHECK-NOT: dmb
364; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
365; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
366
367; CHECK: lduminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
368; CHECK-NOT: dmb
369
370   ret i64 %old
371}
372
373define void @test_atomic_load_umin_i32_noret(i32 %offset) nounwind {
374; CHECK-LABEL: test_atomic_load_umin_i32_noret:
375   atomicrmw umin i32* @var32, i32 %offset seq_cst
376; CHECK-NOT: dmb
377; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
378; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
379
380; CHECK: lduminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
381; CHECK-NOT: dmb
382  ret void
383}
384
385define void @test_atomic_load_umin_i64_noret(i64 %offset) nounwind {
386; CHECK-LABEL: test_atomic_load_umin_i64_noret:
387   atomicrmw umin i64* @var64, i64 %offset seq_cst
388; CHECK-NOT: dmb
389; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
390; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
391
392; CHECK: lduminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
393; CHECK-NOT: dmb
394  ret void
395}
396
397define i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
398; CHECK-LABEL: test_atomic_load_max_i8:
399   %old = atomicrmw max i8* @var8, i8 %offset seq_cst
400; CHECK-NOT: dmb
401; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
402; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
403
404; CHECK: ldsmaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
405; CHECK-NOT: dmb
406
407   ret i8 %old
408}
409
410define i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
411; CHECK-LABEL: test_atomic_load_max_i16:
412   %old = atomicrmw max i16* @var16, i16 %offset seq_cst
413; CHECK-NOT: dmb
414; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
415; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
416
417; CHECK: ldsmaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
418; CHECK-NOT: dmb
419
420   ret i16 %old
421}
422
423define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
424; CHECK-LABEL: test_atomic_load_max_i32:
425   %old = atomicrmw max i32* @var32, i32 %offset seq_cst
426; CHECK-NOT: dmb
427; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
428; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
429
430; CHECK: ldsmaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
431; CHECK-NOT: dmb
432
433   ret i32 %old
434}
435
436define i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
437; CHECK-LABEL: test_atomic_load_max_i64:
438   %old = atomicrmw max i64* @var64, i64 %offset seq_cst
439; CHECK-NOT: dmb
440; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
441; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
442
443; CHECK: ldsmaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
444; CHECK-NOT: dmb
445
446   ret i64 %old
447}
448
449define void @test_atomic_load_max_i32_noret(i32 %offset) nounwind {
450; CHECK-LABEL: test_atomic_load_max_i32_noret:
451   atomicrmw max i32* @var32, i32 %offset seq_cst
452; CHECK-NOT: dmb
453; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
454; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
455
456; CHECK: ldsmaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
457; CHECK-NOT: dmb
458  ret void
459}
460
461define void @test_atomic_load_max_i64_noret(i64 %offset) nounwind {
462; CHECK-LABEL: test_atomic_load_max_i64_noret:
463   atomicrmw max i64* @var64, i64 %offset seq_cst
464; CHECK-NOT: dmb
465; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
466; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
467
468; CHECK: ldsmaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
469; CHECK-NOT: dmb
470  ret void
471}
472
473define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
474; CHECK-LABEL: test_atomic_load_umax_i8:
475   %old = atomicrmw umax i8* @var8, i8 %offset seq_cst
476; CHECK-NOT: dmb
477; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
478; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
479
480; CHECK: ldumaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
481; CHECK-NOT: dmb
482
483   ret i8 %old
484}
485
486define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
487; CHECK-LABEL: test_atomic_load_umax_i16:
488   %old = atomicrmw umax i16* @var16, i16 %offset seq_cst
489; CHECK-NOT: dmb
490; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
491; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
492
493; CHECK: ldumaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
494; CHECK-NOT: dmb
495
496   ret i16 %old
497}
498
499define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
500; CHECK-LABEL: test_atomic_load_umax_i32:
501   %old = atomicrmw umax i32* @var32, i32 %offset seq_cst
502; CHECK-NOT: dmb
503; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
504; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
505
506; CHECK: ldumaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
507; CHECK-NOT: dmb
508
509   ret i32 %old
510}
511
512define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
513; CHECK-LABEL: test_atomic_load_umax_i64:
514   %old = atomicrmw umax i64* @var64, i64 %offset seq_cst
515; CHECK-NOT: dmb
516; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
517; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
518
519; CHECK: ldumaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
520; CHECK-NOT: dmb
521
522   ret i64 %old
523}
524
525define void @test_atomic_load_umax_i32_noret(i32 %offset) nounwind {
526; CHECK-LABEL: test_atomic_load_umax_i32_noret:
527   atomicrmw umax i32* @var32, i32 %offset seq_cst
528; CHECK-NOT: dmb
529; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
530; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
531
532; CHECK: ldumaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
533; CHECK-NOT: dmb
534  ret void
535}
536
537define void @test_atomic_load_umax_i64_noret(i64 %offset) nounwind {
538; CHECK-LABEL: test_atomic_load_umax_i64_noret:
539   atomicrmw umax i64* @var64, i64 %offset seq_cst
540; CHECK-NOT: dmb
541; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
542; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
543
544; CHECK: ldumaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
545; CHECK-NOT: dmb
546  ret void
547}
548
549define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
550; CHECK-LABEL: test_atomic_load_xchg_i8:
551   %old = atomicrmw xchg i8* @var8, i8 %offset seq_cst
552; CHECK-NOT: dmb
553; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
554; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
555
556; CHECK: swpalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
557; CHECK-NOT: dmb
558
559   ret i8 %old
560}
561
562define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
563; CHECK-LABEL: test_atomic_load_xchg_i16:
564   %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst
565; CHECK-NOT: dmb
566; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
567; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
568
569; CHECK: swpalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
570; CHECK-NOT: dmb
571
572   ret i16 %old
573}
574
575define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
576; CHECK-LABEL: test_atomic_load_xchg_i32:
577   %old = atomicrmw xchg i32* @var32, i32 %offset seq_cst
578; CHECK-NOT: dmb
579; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
580; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
581
582; CHECK: swpal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
583; CHECK-NOT: dmb
584
585   ret i32 %old
586}
587
588define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
589; CHECK-LABEL: test_atomic_load_xchg_i64:
590   %old = atomicrmw xchg i64* @var64, i64 %offset seq_cst
591; CHECK-NOT: dmb
592; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
593; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
594
595; CHECK: swpal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
596; CHECK-NOT: dmb
597
598   ret i64 %old
599}
600
601define void @test_atomic_load_xchg_i32_noret(i32 %offset) nounwind {
602; CHECK-LABEL: test_atomic_load_xchg_i32_noret:
603   atomicrmw xchg i32* @var32, i32 %offset seq_cst
604; CHECK-NOT: dmb
605; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
606; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
607
608; CHECK: swpal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
609; CHECK-NOT: dmb
610
611   ret void
612}
613
614define void @test_atomic_load_xchg_i64_noret(i64 %offset) nounwind {
615; CHECK-LABEL: test_atomic_load_xchg_i64_noret:
616   atomicrmw xchg i64* @var64, i64 %offset seq_cst
617; CHECK-NOT: dmb
618; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
619; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
620
621; CHECK: swpal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
622; CHECK-NOT: dmb
623
624   ret void
625}
626
627define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
628; CHECK-LABEL: test_atomic_cmpxchg_i8:
629   %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
630   %old = extractvalue { i8, i1 } %pair, 0
631
632; CHECK-NOT: dmb
633; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
634; CHECK-NEXT: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
635; CHECK-NEXT: casab w0, w1, [x[[ADDR]]]
636; CHECK-NEXT: ret
637
638   ret i8 %old
639}
640
641define i1 @test_atomic_cmpxchg_i8_1(i8 %wanted, i8 %new) nounwind {
642; CHECK-LABEL: test_atomic_cmpxchg_i8_1:
643   %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
644   %success = extractvalue { i8, i1 } %pair, 1
645
646; CHECK-NOT: dmb
647; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
648; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
649
650; CHECK: casab w[[NEW:[0-9]+]], w1, [x[[ADDR]]]
651; CHECK-NEXT: cmp w[[NEW]], w0, uxtb
652; CHECK-NEXT: cset w0, eq
653; CHECK-NEXT: ret
654   ret i1 %success
655}
656
657define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
658; CHECK-LABEL: test_atomic_cmpxchg_i16:
659   %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new acquire acquire
660   %old = extractvalue { i16, i1 } %pair, 0
661
662; CHECK-NOT: dmb
663; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
664; CHECK-NEXT: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
665; CHECK-NEXT: casah w0, w1, [x[[ADDR]]]
666; CHECK-NEXT: ret
667
668   ret i16 %old
669}
670
671define i1 @test_atomic_cmpxchg_i16_1(i16 %wanted, i16 %new) nounwind {
672; CHECK-LABEL: test_atomic_cmpxchg_i16_1:
673   %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new acquire acquire
674   %success = extractvalue { i16, i1 } %pair, 1
675
676; CHECK-NOT: dmb
677; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
678; CHECK-NEXT: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
679
680; CHECK: casah w[[NEW:[0-9]+]], w1, [x[[ADDR]]]
681; CHECK-NEXT: cmp w[[NEW]], w0, uxth
682; CHECK-NEXT: cset w0, eq
683; CHECK-NEXT: ret
684
685   ret i1 %success
686}
687
688define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
689; CHECK-LABEL: test_atomic_cmpxchg_i32:
690   %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new acquire acquire
691   %old = extractvalue { i32, i1 } %pair, 0
692
693; CHECK-NOT: dmb
694; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
695; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
696
697; CHECK: casa w0, w1, [x[[ADDR]]]
698; CHECK-NOT: dmb
699
700   ret i32 %old
701}
702
703define i64 @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
704; CHECK-LABEL: test_atomic_cmpxchg_i64:
705   %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new acquire acquire
706   %old = extractvalue { i64, i1 } %pair, 0
707
708; CHECK-NOT: dmb
709; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
710; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
711
712; CHECK: casa x0, x1, [x[[ADDR]]]
713; CHECK-NOT: dmb
714
715   ret i64 %old
716}
717
718define i128 @test_atomic_cmpxchg_i128(i128 %wanted, i128 %new) nounwind {
719; CHECK-LABEL: test_atomic_cmpxchg_i128:
720   %pair = cmpxchg i128* @var128, i128 %wanted, i128 %new acquire acquire
721   %old = extractvalue { i128, i1 } %pair, 0
722
723; CHECK-NOT: dmb
724; CHECK: adrp [[TMPADDR:x[0-9]+]], var128
725; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128
726
727; CHECK: caspa x0, x1, x2, x3, [x[[ADDR]]]
728; CHECK-NOT: dmb
729
730   ret i128 %old
731}
732
733define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
734; CHECK-LABEL: test_atomic_load_sub_i8:
735  %old = atomicrmw sub i8* @var8, i8 %offset seq_cst
736; CHECK-NOT: dmb
737; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
738; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
739; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
740
741; CHECK: ldaddalb w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
742; CHECK-NOT: dmb
743
744  ret i8 %old
745}
746
747define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
748; CHECK-LABEL: test_atomic_load_sub_i16:
749  %old = atomicrmw sub i16* @var16, i16 %offset seq_cst
750; CHECK-NOT: dmb
751; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
752; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
753; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
754
755; CHECK: ldaddalh w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
756; CHECK-NOT: dmb
757
758  ret i16 %old
759}
760
761define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
762; CHECK-LABEL: test_atomic_load_sub_i32:
763  %old = atomicrmw sub i32* @var32, i32 %offset seq_cst
764; CHECK-NOT: dmb
765; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
766; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
767; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
768
769; CHECK: ldaddal w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
770; CHECK-NOT: dmb
771
772  ret i32 %old
773}
774
775define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
776; CHECK-LABEL: test_atomic_load_sub_i64:
777  %old = atomicrmw sub i64* @var64, i64 %offset seq_cst
778; CHECK-NOT: dmb
779; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]]
780; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
781; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
782
783; CHECK: ldaddal x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]]
784; CHECK-NOT: dmb
785
786  ret i64 %old
787}
788
789define void @test_atomic_load_sub_i32_noret(i32 %offset) nounwind {
790; CHECK-LABEL: test_atomic_load_sub_i32_noret:
791  atomicrmw sub i32* @var32, i32 %offset seq_cst
792; CHECK-NOT: dmb
793; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
794; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
795; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
796
797; CHECK: ldaddal w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
798; CHECK-NOT: dmb
799
800  ret void
801}
802
803define void @test_atomic_load_sub_i64_noret(i64 %offset) nounwind {
804; CHECK-LABEL: test_atomic_load_sub_i64_noret:
805  atomicrmw sub i64* @var64, i64 %offset seq_cst
806; CHECK-NOT: dmb
807; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]]
808; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
809; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
810
811; CHECK: ldaddal x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]]
812; CHECK-NOT: dmb
813
814  ret void
815}
816
817define i8 @test_atomic_load_sub_i8_neg_imm() nounwind {
818; CHECK-LABEL: test_atomic_load_sub_i8_neg_imm:
819  %old = atomicrmw sub i8* @var8, i8 -1 seq_cst
820
821; CHECK-NOT: dmb
822; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
823; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
824; CHECK: orr w[[IMM:[0-9]+]], wzr, #0x1
825; CHECK: ldaddalb w[[IMM]], w[[NEW:[0-9]+]], [x[[ADDR]]]
826; CHECK-NOT: dmb
827
828  ret i8 %old
829}
830
831define i16 @test_atomic_load_sub_i16_neg_imm() nounwind {
832; CHECK-LABEL: test_atomic_load_sub_i16_neg_imm:
833  %old = atomicrmw sub i16* @var16, i16 -1 seq_cst
834
835; CHECK-NOT: dmb
836; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
837; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
838; CHECK: orr w[[IMM:[0-9]+]], wzr, #0x1
839; CHECK: ldaddalh w[[IMM]], w[[NEW:[0-9]+]], [x[[ADDR]]]
840; CHECK-NOT: dmb
841
842  ret i16 %old
843}
844
845define i32 @test_atomic_load_sub_i32_neg_imm() nounwind {
846; CHECK-LABEL: test_atomic_load_sub_i32_neg_imm:
847  %old = atomicrmw sub i32* @var32, i32 -1 seq_cst
848
849; CHECK-NOT: dmb
850; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
851; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
852; CHECK: orr w[[IMM:[0-9]+]], wzr, #0x1
853; CHECK: ldaddal w[[IMM]], w[[NEW:[0-9]+]], [x[[ADDR]]]
854; CHECK-NOT: dmb
855
856  ret i32 %old
857}
858
859define i64 @test_atomic_load_sub_i64_neg_imm() nounwind {
860; CHECK-LABEL: test_atomic_load_sub_i64_neg_imm:
861  %old = atomicrmw sub i64* @var64, i64 -1 seq_cst
862
863; CHECK-NOT: dmb
864; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
865; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
866; CHECK: orr w[[IMM:[0-9]+]], wzr, #0x1
867; CHECK: ldaddal x[[IMM]], x[[NEW:[0-9]+]], [x[[ADDR]]]
868; CHECK-NOT: dmb
869
870  ret i64 %old
871}
872
873define i8 @test_atomic_load_sub_i8_neg_arg(i8 %offset) nounwind {
874; CHECK-LABEL: test_atomic_load_sub_i8_neg_arg:
875  %neg = sub i8 0, %offset
876  %old = atomicrmw sub i8* @var8, i8 %neg seq_cst
877
878; CHECK-NOT: dmb
879; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
880; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
881; CHECK: ldaddalb w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
882; CHECK-NOT: dmb
883
884  ret i8 %old
885}
886
887define i16 @test_atomic_load_sub_i16_neg_arg(i16 %offset) nounwind {
888; CHECK-LABEL: test_atomic_load_sub_i16_neg_arg:
889  %neg = sub i16 0, %offset
890  %old = atomicrmw sub i16* @var16, i16 %neg seq_cst
891
892; CHECK-NOT: dmb
893; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
894; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
895; CHECK: ldaddalh w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
896; CHECK-NOT: dmb
897
898  ret i16 %old
899}
900
901define i32 @test_atomic_load_sub_i32_neg_arg(i32 %offset) nounwind {
902; CHECK-LABEL: test_atomic_load_sub_i32_neg_arg:
903  %neg = sub i32 0, %offset
904  %old = atomicrmw sub i32* @var32, i32 %neg seq_cst
905
906; CHECK-NOT: dmb
907; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
908; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
909; CHECK: ldaddal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
910; CHECK-NOT: dmb
911
912  ret i32 %old
913}
914
915define i64 @test_atomic_load_sub_i64_neg_arg(i64 %offset) nounwind {
916; CHECK-LABEL: test_atomic_load_sub_i64_neg_arg:
917  %neg = sub i64 0, %offset
918  %old = atomicrmw sub i64* @var64, i64 %neg seq_cst
919
920; CHECK-NOT: dmb
921; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
922; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
923; CHECK: ldaddal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
924; CHECK-NOT: dmb
925
926  ret i64 %old
927}
928
929define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
930; CHECK-LABEL: test_atomic_load_and_i8:
931  %old = atomicrmw and i8* @var8, i8 %offset seq_cst
932; CHECK-NOT: dmb
933; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
934; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
935; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
936
937; CHECK: ldclralb w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
938; CHECK-NOT: dmb
939  ret i8 %old
940}
941
942define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
943; CHECK-LABEL: test_atomic_load_and_i16:
944  %old = atomicrmw and i16* @var16, i16 %offset seq_cst
945; CHECK-NOT: dmb
946; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
947; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
948; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
949
950; CHECK: ldclralh w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
951; CHECK-NOT: dmb
952  ret i16 %old
953}
954
955define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
956; CHECK-LABEL: test_atomic_load_and_i32:
957  %old = atomicrmw and i32* @var32, i32 %offset seq_cst
958; CHECK-NOT: dmb
959; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
960; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
961; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
962
963; CHECK: ldclral w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
964; CHECK-NOT: dmb
965  ret i32 %old
966}
967
968define i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
969; CHECK-LABEL: test_atomic_load_and_i64:
970  %old = atomicrmw and i64* @var64, i64 %offset seq_cst
971; CHECK-NOT: dmb
972; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]]
973; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
974; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
975
976; CHECK: ldclral x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]]
977; CHECK-NOT: dmb
978  ret i64 %old
979}
980
981define i8 @test_atomic_load_and_i8_inv_imm() nounwind {
982; CHECK-LABEL: test_atomic_load_and_i8_inv_imm:
983  %old = atomicrmw and i8* @var8, i8 -2 seq_cst
984; CHECK-NOT: dmb
985; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
986; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
987; CHECK: orr w[[CONST:[0-9]+]], wzr, #0x1
988; CHECK: ldclralb w[[CONST]], w[[NEW:[0-9]+]], [x[[ADDR]]]
989; CHECK-NOT: dmb
990  ret i8 %old
991}
992
993define i16 @test_atomic_load_and_i16_inv_imm() nounwind {
994; CHECK-LABEL: test_atomic_load_and_i16_inv_imm:
995  %old = atomicrmw and i16* @var16, i16 -2 seq_cst
996; CHECK-NOT: dmb
997; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
998; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
999; CHECK: orr w[[CONST:[0-9]+]], wzr, #0x1
1000; CHECK: ldclralh w[[CONST]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1001; CHECK-NOT: dmb
1002  ret i16 %old
1003}
1004
1005define i32 @test_atomic_load_and_i32_inv_imm() nounwind {
1006; CHECK-LABEL: test_atomic_load_and_i32_inv_imm:
1007  %old = atomicrmw and i32* @var32, i32 -2 seq_cst
1008; CHECK-NOT: dmb
1009; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1010; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1011; CHECK: orr w[[CONST:[0-9]+]], wzr, #0x1
1012; CHECK: ldclral w[[CONST]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1013; CHECK-NOT: dmb
1014  ret i32 %old
1015}
1016
1017define i64 @test_atomic_load_and_i64_inv_imm() nounwind {
1018; CHECK-LABEL: test_atomic_load_and_i64_inv_imm:
1019  %old = atomicrmw and i64* @var64, i64 -2 seq_cst
1020; CHECK-NOT: dmb
1021; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1022; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1023; CHECK: orr w[[CONST:[0-9]+]], wzr, #0x1
1024; CHECK: ldclral x[[CONST]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1025; CHECK-NOT: dmb
1026  ret i64 %old
1027}
1028
1029define i8 @test_atomic_load_and_i8_inv_arg(i8 %offset) nounwind {
1030; CHECK-LABEL: test_atomic_load_and_i8_inv_arg:
1031  %inv = xor i8 %offset, -1
1032  %old = atomicrmw and i8* @var8, i8 %inv seq_cst
1033; CHECK-NOT: dmb
1034; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1035; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1036; CHECK: ldclralb w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
1037; CHECK-NOT: dmb
1038  ret i8 %old
1039}
1040
1041define i16 @test_atomic_load_and_i16_inv_arg(i16 %offset) nounwind {
1042; CHECK-LABEL: test_atomic_load_and_i16_inv_arg:
1043  %inv = xor i16 %offset, -1
1044  %old = atomicrmw and i16* @var16, i16 %inv seq_cst
1045; CHECK-NOT: dmb
1046; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1047; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1048; CHECK: ldclralh w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
1049; CHECK-NOT: dmb
1050  ret i16 %old
1051}
1052
1053define i32 @test_atomic_load_and_i32_inv_arg(i32 %offset) nounwind {
1054; CHECK-LABEL: test_atomic_load_and_i32_inv_arg:
1055  %inv = xor i32 %offset, -1
1056  %old = atomicrmw and i32* @var32, i32 %inv seq_cst
1057; CHECK-NOT: dmb
1058; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1059; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1060; CHECK: ldclral w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
1061; CHECK-NOT: dmb
1062  ret i32 %old
1063}
1064
1065define i64 @test_atomic_load_and_i64_inv_arg(i64 %offset) nounwind {
1066; CHECK-LABEL: test_atomic_load_and_i64_inv_arg:
1067  %inv = xor i64 %offset, -1
1068  %old = atomicrmw and i64* @var64, i64 %inv seq_cst
1069; CHECK-NOT: dmb
1070; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1071; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1072; CHECK: ldclral x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
1073; CHECK-NOT: dmb
1074  ret i64 %old
1075}
1076
1077define void @test_atomic_load_and_i32_noret(i32 %offset) nounwind {
1078; CHECK-LABEL: test_atomic_load_and_i32_noret:
1079  atomicrmw and i32* @var32, i32 %offset seq_cst
1080; CHECK-NOT: dmb
1081; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1082; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1083; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1084
1085; CHECK: ldclral w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1086; CHECK-NOT: dmb
1087  ret void
1088}
1089
1090define void @test_atomic_load_and_i64_noret(i64 %offset) nounwind {
1091; CHECK-LABEL: test_atomic_load_and_i64_noret:
1092  atomicrmw and i64* @var64, i64 %offset seq_cst
1093; CHECK-NOT: dmb
1094; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]]
1095; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1096; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1097
1098; CHECK: ldclral x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1099; CHECK-NOT: dmb
1100  ret void
1101}
1102
1103define i8 @test_atomic_load_add_i8_acq_rel(i8 %offset) nounwind {
1104; CHECK-LABEL: test_atomic_load_add_i8_acq_rel:
1105   %old = atomicrmw add i8* @var8, i8 %offset acq_rel
1106; CHECK-NOT: dmb
1107; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1108; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1109
1110; CHECK: ldaddalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1111; CHECK-NOT: dmb
1112
1113   ret i8 %old
1114}
1115
1116define i16 @test_atomic_load_add_i16_acq_rel(i16 %offset) nounwind {
1117; CHECK-LABEL: test_atomic_load_add_i16_acq_rel:
1118   %old = atomicrmw add i16* @var16, i16 %offset acq_rel
1119; CHECK-NOT: dmb
1120; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1121; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1122
1123; CHECK: ldaddalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1124; CHECK-NOT: dmb
1125
1126   ret i16 %old
1127}
1128
1129define i32 @test_atomic_load_add_i32_acq_rel(i32 %offset) nounwind {
1130; CHECK-LABEL: test_atomic_load_add_i32_acq_rel:
1131   %old = atomicrmw add i32* @var32, i32 %offset acq_rel
1132; CHECK-NOT: dmb
1133; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1134; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1135
1136; CHECK: ldaddal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1137; CHECK-NOT: dmb
1138
1139   ret i32 %old
1140}
1141
1142define i64 @test_atomic_load_add_i64_acq_rel(i64 %offset) nounwind {
1143; CHECK-LABEL: test_atomic_load_add_i64_acq_rel:
1144   %old = atomicrmw add i64* @var64, i64 %offset acq_rel
1145; CHECK-NOT: dmb
1146; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1147; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1148
1149; CHECK: ldaddal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1150; CHECK-NOT: dmb
1151
1152   ret i64 %old
1153}
1154
1155define void @test_atomic_load_add_i32_noret_acq_rel(i32 %offset) nounwind {
1156; CHECK-LABEL: test_atomic_load_add_i32_noret_acq_rel:
1157   atomicrmw add i32* @var32, i32 %offset acq_rel
1158; CHECK-NOT: dmb
1159; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1160; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1161
1162; CHECK: ldaddal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
1163; CHECK-NOT: dmb
1164  ret void
1165}
1166
1167define void @test_atomic_load_add_i64_noret_acq_rel(i64 %offset) nounwind {
1168; CHECK-LABEL: test_atomic_load_add_i64_noret_acq_rel:
1169   atomicrmw add i64* @var64, i64 %offset acq_rel
1170; CHECK-NOT: dmb
1171; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1172; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1173
1174; CHECK: ldaddal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
1175; CHECK-NOT: dmb
1176  ret void
1177}
1178
1179define i8 @test_atomic_load_add_i8_acquire(i8 %offset) nounwind {
1180; CHECK-LABEL: test_atomic_load_add_i8_acquire:
1181   %old = atomicrmw add i8* @var8, i8 %offset acquire
1182; CHECK-NOT: dmb
1183; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1184; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1185
1186; CHECK: ldaddab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1187; CHECK-NOT: dmb
1188
1189   ret i8 %old
1190}
1191
1192define i16 @test_atomic_load_add_i16_acquire(i16 %offset) nounwind {
1193; CHECK-LABEL: test_atomic_load_add_i16_acquire:
1194   %old = atomicrmw add i16* @var16, i16 %offset acquire
1195; CHECK-NOT: dmb
1196; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1197; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1198
1199; CHECK: ldaddah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1200; CHECK-NOT: dmb
1201
1202   ret i16 %old
1203}
1204
1205define i32 @test_atomic_load_add_i32_acquire(i32 %offset) nounwind {
1206; CHECK-LABEL: test_atomic_load_add_i32_acquire:
1207   %old = atomicrmw add i32* @var32, i32 %offset acquire
1208; CHECK-NOT: dmb
1209; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1210; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1211
1212; CHECK: ldadda w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1213; CHECK-NOT: dmb
1214
1215   ret i32 %old
1216}
1217
1218define i64 @test_atomic_load_add_i64_acquire(i64 %offset) nounwind {
1219; CHECK-LABEL: test_atomic_load_add_i64_acquire:
1220   %old = atomicrmw add i64* @var64, i64 %offset acquire
1221; CHECK-NOT: dmb
1222; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1223; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1224
1225; CHECK: ldadda x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1226; CHECK-NOT: dmb
1227
1228   ret i64 %old
1229}
1230
1231define void @test_atomic_load_add_i32_noret_acquire(i32 %offset) nounwind {
1232; CHECK-LABEL: test_atomic_load_add_i32_noret_acquire:
1233   atomicrmw add i32* @var32, i32 %offset acquire
1234; CHECK-NOT: dmb
1235; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1236; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1237
1238; CHECK: ldadda w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
1239; CHECK-NOT: dmb
1240  ret void
1241}
1242
1243define void @test_atomic_load_add_i64_noret_acquire(i64 %offset) nounwind {
1244; CHECK-LABEL: test_atomic_load_add_i64_noret_acquire:
1245   atomicrmw add i64* @var64, i64 %offset acquire
1246; CHECK-NOT: dmb
1247; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1248; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1249
1250; CHECK: ldadda x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
1251; CHECK-NOT: dmb
1252  ret void
1253}
1254
1255define i8 @test_atomic_load_add_i8_monotonic(i8 %offset) nounwind {
1256; CHECK-LABEL: test_atomic_load_add_i8_monotonic:
1257   %old = atomicrmw add i8* @var8, i8 %offset monotonic
1258; CHECK-NOT: dmb
1259; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1260; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1261
1262; CHECK: ldaddb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1263; CHECK-NOT: dmb
1264
1265   ret i8 %old
1266}
1267
1268define i16 @test_atomic_load_add_i16_monotonic(i16 %offset) nounwind {
1269; CHECK-LABEL: test_atomic_load_add_i16_monotonic:
1270   %old = atomicrmw add i16* @var16, i16 %offset monotonic
1271; CHECK-NOT: dmb
1272; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1273; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1274
1275; CHECK: ldaddh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1276; CHECK-NOT: dmb
1277
1278   ret i16 %old
1279}
1280
1281define i32 @test_atomic_load_add_i32_monotonic(i32 %offset) nounwind {
1282; CHECK-LABEL: test_atomic_load_add_i32_monotonic:
1283   %old = atomicrmw add i32* @var32, i32 %offset monotonic
1284; CHECK-NOT: dmb
1285; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1286; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1287
1288; CHECK: ldadd w[[OLD:[0-9]+]], w[[NEW:[0-9,a-z]+]], [x[[ADDR]]]
1289; CHECK-NOT: dmb
1290
1291   ret i32 %old
1292}
1293
1294define i64 @test_atomic_load_add_i64_monotonic(i64 %offset) nounwind {
1295; CHECK-LABEL: test_atomic_load_add_i64_monotonic:
1296   %old = atomicrmw add i64* @var64, i64 %offset monotonic
1297; CHECK-NOT: dmb
1298; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1299; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1300
1301; CHECK: ldadd x[[OLD:[0-9]+]], x[[NEW:[0-9,a-z]+]], [x[[ADDR]]]
1302; CHECK-NOT: dmb
1303
1304   ret i64 %old
1305}
1306
1307define void @test_atomic_load_add_i32_noret_monotonic(i32 %offset) nounwind {
1308; CHECK-LABEL: test_atomic_load_add_i32_noret_monotonic:
1309   atomicrmw add i32* @var32, i32 %offset monotonic
1310; CHECK-NOT: dmb
1311; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1312; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1313
1314; CHECK: stadd w0, [x[[ADDR]]]
1315; CHECK-NOT: dmb
1316  ret void
1317}
1318
1319define void @test_atomic_load_add_i64_noret_monotonic(i64 %offset) nounwind {
1320; CHECK-LABEL: test_atomic_load_add_i64_noret_monotonic:
1321   atomicrmw add i64* @var64, i64 %offset monotonic
1322; CHECK-NOT: dmb
1323; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1324; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1325
1326; CHECK: stadd x0, [x[[ADDR]]]
1327; CHECK-NOT: dmb
1328  ret void
1329}
1330
1331define i8 @test_atomic_load_add_i8_release(i8 %offset) nounwind {
1332; CHECK-LABEL: test_atomic_load_add_i8_release:
1333   %old = atomicrmw add i8* @var8, i8 %offset release
1334; CHECK-NOT: dmb
1335; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1336; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1337
1338; CHECK: ldaddlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1339; CHECK-NOT: dmb
1340
1341   ret i8 %old
1342}
1343
1344define i16 @test_atomic_load_add_i16_release(i16 %offset) nounwind {
1345; CHECK-LABEL: test_atomic_load_add_i16_release:
1346   %old = atomicrmw add i16* @var16, i16 %offset release
1347; CHECK-NOT: dmb
1348; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1349; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1350
1351; CHECK: ldaddlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1352; CHECK-NOT: dmb
1353
1354   ret i16 %old
1355}
1356
1357define i32 @test_atomic_load_add_i32_release(i32 %offset) nounwind {
1358; CHECK-LABEL: test_atomic_load_add_i32_release:
1359   %old = atomicrmw add i32* @var32, i32 %offset release
1360; CHECK-NOT: dmb
1361; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1362; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1363
1364; CHECK: ldaddl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1365; CHECK-NOT: dmb
1366
1367   ret i32 %old
1368}
1369
1370define i64 @test_atomic_load_add_i64_release(i64 %offset) nounwind {
1371; CHECK-LABEL: test_atomic_load_add_i64_release:
1372   %old = atomicrmw add i64* @var64, i64 %offset release
1373; CHECK-NOT: dmb
1374; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1375; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1376
1377; CHECK: ldaddl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1378; CHECK-NOT: dmb
1379
1380   ret i64 %old
1381}
1382
1383define void @test_atomic_load_add_i32_noret_release(i32 %offset) nounwind {
1384; CHECK-LABEL: test_atomic_load_add_i32_noret_release:
1385   atomicrmw add i32* @var32, i32 %offset release
1386; CHECK-NOT: dmb
1387; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1388; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1389
1390; CHECK: staddl w0, [x[[ADDR]]]
1391; CHECK-NOT: dmb
1392  ret void
1393}
1394
1395define void @test_atomic_load_add_i64_noret_release(i64 %offset) nounwind {
1396; CHECK-LABEL: test_atomic_load_add_i64_noret_release:
1397   atomicrmw add i64* @var64, i64 %offset release
1398; CHECK-NOT: dmb
1399; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1400; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1401
1402; CHECK: staddl x0, [x[[ADDR]]]
1403; CHECK-NOT: dmb
1404  ret void
1405}
1406
1407define i8 @test_atomic_load_add_i8_seq_cst(i8 %offset) nounwind {
1408; CHECK-LABEL: test_atomic_load_add_i8_seq_cst:
1409   %old = atomicrmw add i8* @var8, i8 %offset seq_cst
1410; CHECK-NOT: dmb
1411; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1412; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1413
1414; CHECK: ldaddalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1415; CHECK-NOT: dmb
1416
1417   ret i8 %old
1418}
1419
1420define i16 @test_atomic_load_add_i16_seq_cst(i16 %offset) nounwind {
1421; CHECK-LABEL: test_atomic_load_add_i16_seq_cst:
1422   %old = atomicrmw add i16* @var16, i16 %offset seq_cst
1423; CHECK-NOT: dmb
1424; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1425; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1426
1427; CHECK: ldaddalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1428; CHECK-NOT: dmb
1429
1430   ret i16 %old
1431}
1432
1433define i32 @test_atomic_load_add_i32_seq_cst(i32 %offset) nounwind {
1434; CHECK-LABEL: test_atomic_load_add_i32_seq_cst:
1435   %old = atomicrmw add i32* @var32, i32 %offset seq_cst
1436; CHECK-NOT: dmb
1437; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1438; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1439
1440; CHECK: ldaddal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1441; CHECK-NOT: dmb
1442
1443   ret i32 %old
1444}
1445
1446define i64 @test_atomic_load_add_i64_seq_cst(i64 %offset) nounwind {
1447; CHECK-LABEL: test_atomic_load_add_i64_seq_cst:
1448   %old = atomicrmw add i64* @var64, i64 %offset seq_cst
1449; CHECK-NOT: dmb
1450; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1451; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1452
1453; CHECK: ldaddal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1454; CHECK-NOT: dmb
1455
1456   ret i64 %old
1457}
1458
1459define void @test_atomic_load_add_i32_noret_seq_cst(i32 %offset) nounwind {
1460; CHECK-LABEL: test_atomic_load_add_i32_noret_seq_cst:
1461   atomicrmw add i32* @var32, i32 %offset seq_cst
1462; CHECK-NOT: dmb
1463; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1464; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1465
1466; CHECK: ldaddal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
1467; CHECK-NOT: dmb
1468  ret void
1469}
1470
1471define void @test_atomic_load_add_i64_noret_seq_cst(i64 %offset) nounwind {
1472; CHECK-LABEL: test_atomic_load_add_i64_noret_seq_cst:
1473   atomicrmw add i64* @var64, i64 %offset seq_cst
1474; CHECK-NOT: dmb
1475; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1476; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1477
1478; CHECK: ldaddal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
1479; CHECK-NOT: dmb
1480  ret void
1481}
1482
1483define i8 @test_atomic_load_and_i8_acq_rel(i8 %offset) nounwind {
1484; CHECK-LABEL: test_atomic_load_and_i8_acq_rel:
1485  %old = atomicrmw and i8* @var8, i8 %offset acq_rel
1486; CHECK-NOT: dmb
1487; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1488; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1489; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1490
1491; CHECK: ldclralb w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1492; CHECK-NOT: dmb
1493  ret i8 %old
1494}
1495
1496define i16 @test_atomic_load_and_i16_acq_rel(i16 %offset) nounwind {
1497; CHECK-LABEL: test_atomic_load_and_i16_acq_rel:
1498  %old = atomicrmw and i16* @var16, i16 %offset acq_rel
1499; CHECK-NOT: dmb
1500; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1501; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1502; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1503
1504; CHECK: ldclralh w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1505; CHECK-NOT: dmb
1506  ret i16 %old
1507}
1508
1509define i32 @test_atomic_load_and_i32_acq_rel(i32 %offset) nounwind {
1510; CHECK-LABEL: test_atomic_load_and_i32_acq_rel:
1511  %old = atomicrmw and i32* @var32, i32 %offset acq_rel
1512; CHECK-NOT: dmb
1513; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1514; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1515; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1516
1517; CHECK: ldclral w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1518; CHECK-NOT: dmb
1519  ret i32 %old
1520}
1521
1522define i64 @test_atomic_load_and_i64_acq_rel(i64 %offset) nounwind {
1523; CHECK-LABEL: test_atomic_load_and_i64_acq_rel:
1524  %old = atomicrmw and i64* @var64, i64 %offset acq_rel
1525; CHECK-NOT: dmb
1526; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]]
1527; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1528; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1529
1530; CHECK: ldclral x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1531; CHECK-NOT: dmb
1532  ret i64 %old
1533}
1534
1535define void @test_atomic_load_and_i32_noret_acq_rel(i32 %offset) nounwind {
1536; CHECK-LABEL: test_atomic_load_and_i32_noret_acq_rel:
1537  atomicrmw and i32* @var32, i32 %offset acq_rel
1538; CHECK-NOT: dmb
1539; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1540; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1541; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1542
1543; CHECK: ldclral w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1544; CHECK-NOT: dmb
1545  ret void
1546}
1547
1548define void @test_atomic_load_and_i64_noret_acq_rel(i64 %offset) nounwind {
1549; CHECK-LABEL: test_atomic_load_and_i64_noret_acq_rel:
1550  atomicrmw and i64* @var64, i64 %offset acq_rel
1551; CHECK-NOT: dmb
1552; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]]
1553; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1554; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1555
1556; CHECK: ldclral x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1557; CHECK-NOT: dmb
1558  ret void
1559}
1560
1561define i8 @test_atomic_load_and_i8_acquire(i8 %offset) nounwind {
1562; CHECK-LABEL: test_atomic_load_and_i8_acquire:
1563  %old = atomicrmw and i8* @var8, i8 %offset acquire
1564; CHECK-NOT: dmb
1565; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1566; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1567; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1568
1569; CHECK: ldclrab w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1570; CHECK-NOT: dmb
1571  ret i8 %old
1572}
1573
1574define i16 @test_atomic_load_and_i16_acquire(i16 %offset) nounwind {
1575; CHECK-LABEL: test_atomic_load_and_i16_acquire:
1576  %old = atomicrmw and i16* @var16, i16 %offset acquire
1577; CHECK-NOT: dmb
1578; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1579; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1580; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1581
1582; CHECK: ldclrah w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1583; CHECK-NOT: dmb
1584  ret i16 %old
1585}
1586
1587define i32 @test_atomic_load_and_i32_acquire(i32 %offset) nounwind {
1588; CHECK-LABEL: test_atomic_load_and_i32_acquire:
1589  %old = atomicrmw and i32* @var32, i32 %offset acquire
1590; CHECK-NOT: dmb
1591; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1592; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1593; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1594
1595; CHECK: ldclra w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1596; CHECK-NOT: dmb
1597  ret i32 %old
1598}
1599
1600define i64 @test_atomic_load_and_i64_acquire(i64 %offset) nounwind {
1601; CHECK-LABEL: test_atomic_load_and_i64_acquire:
1602  %old = atomicrmw and i64* @var64, i64 %offset acquire
1603; CHECK-NOT: dmb
1604; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]]
1605; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1606; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1607
1608; CHECK: ldclra x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1609; CHECK-NOT: dmb
1610  ret i64 %old
1611}
1612
1613define void @test_atomic_load_and_i32_noret_acquire(i32 %offset) nounwind {
1614; CHECK-LABEL: test_atomic_load_and_i32_noret_acquire:
1615  atomicrmw and i32* @var32, i32 %offset acquire
1616; CHECK-NOT: dmb
1617; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1618; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1619; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1620
1621; CHECK: ldclra w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1622; CHECK-NOT: dmb
1623  ret void
1624}
1625
1626define void @test_atomic_load_and_i64_noret_acquire(i64 %offset) nounwind {
1627; CHECK-LABEL: test_atomic_load_and_i64_noret_acquire:
1628  atomicrmw and i64* @var64, i64 %offset acquire
1629; CHECK-NOT: dmb
1630; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]]
1631; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1632; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1633
1634; CHECK: ldclra x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1635; CHECK-NOT: dmb
1636  ret void
1637}
1638
1639define i8 @test_atomic_load_and_i8_monotonic(i8 %offset) nounwind {
1640; CHECK-LABEL: test_atomic_load_and_i8_monotonic:
1641  %old = atomicrmw and i8* @var8, i8 %offset monotonic
1642; CHECK-NOT: dmb
1643; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1644; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1645; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1646
1647; CHECK: ldclrb w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1648; CHECK-NOT: dmb
1649  ret i8 %old
1650}
1651
1652define i16 @test_atomic_load_and_i16_monotonic(i16 %offset) nounwind {
1653; CHECK-LABEL: test_atomic_load_and_i16_monotonic:
1654  %old = atomicrmw and i16* @var16, i16 %offset monotonic
1655; CHECK-NOT: dmb
1656; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1657; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1658; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1659
1660; CHECK: ldclrh w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1661; CHECK-NOT: dmb
1662  ret i16 %old
1663}
1664
1665define i32 @test_atomic_load_and_i32_monotonic(i32 %offset) nounwind {
1666; CHECK-LABEL: test_atomic_load_and_i32_monotonic:
1667  %old = atomicrmw and i32* @var32, i32 %offset monotonic
1668; CHECK-NOT: dmb
1669; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1670; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1671; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1672
1673; CHECK: ldclr w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1674; CHECK-NOT: dmb
1675  ret i32 %old
1676}
1677
1678define i64 @test_atomic_load_and_i64_monotonic(i64 %offset) nounwind {
1679; CHECK-LABEL: test_atomic_load_and_i64_monotonic:
1680  %old = atomicrmw and i64* @var64, i64 %offset monotonic
1681; CHECK-NOT: dmb
1682; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]]
1683; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1684; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1685
1686; CHECK: ldclr x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1687; CHECK-NOT: dmb
1688  ret i64 %old
1689}
1690
1691define void @test_atomic_load_and_i32_noret_monotonic(i32 %offset) nounwind {
1692; CHECK-LABEL: test_atomic_load_and_i32_noret_monotonic:
1693  atomicrmw and i32* @var32, i32 %offset monotonic
1694; CHECK-NOT: dmb
1695; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1696; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1697; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1698
1699; CHECK: stclr w[[NEW:[0-9]+]], [x[[ADDR]]]
1700; CHECK-NOT: dmb
1701  ret void
1702}
1703
1704define void @test_atomic_load_and_i64_noret_monotonic(i64 %offset) nounwind {
1705; CHECK-LABEL: test_atomic_load_and_i64_noret_monotonic:
1706  atomicrmw and i64* @var64, i64 %offset monotonic
1707; CHECK-NOT: dmb
1708; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]]
1709; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1710; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1711
1712; CHECK: stclr x[[NEW:[0-9]+]], [x[[ADDR]]]
1713; CHECK-NOT: dmb
1714  ret void
1715}
1716
1717define i8 @test_atomic_load_and_i8_release(i8 %offset) nounwind {
1718; CHECK-LABEL: test_atomic_load_and_i8_release:
1719  %old = atomicrmw and i8* @var8, i8 %offset release
1720; CHECK-NOT: dmb
1721; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1722; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1723; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1724
1725; CHECK: ldclrlb w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1726; CHECK-NOT: dmb
1727  ret i8 %old
1728}
1729
1730define i16 @test_atomic_load_and_i16_release(i16 %offset) nounwind {
1731; CHECK-LABEL: test_atomic_load_and_i16_release:
1732  %old = atomicrmw and i16* @var16, i16 %offset release
1733; CHECK-NOT: dmb
1734; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1735; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1736; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1737
1738; CHECK: ldclrlh w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1739; CHECK-NOT: dmb
1740  ret i16 %old
1741}
1742
1743define i32 @test_atomic_load_and_i32_release(i32 %offset) nounwind {
1744; CHECK-LABEL: test_atomic_load_and_i32_release:
1745  %old = atomicrmw and i32* @var32, i32 %offset release
1746; CHECK-NOT: dmb
1747; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1748; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1749; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1750
1751; CHECK: ldclrl w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1752; CHECK-NOT: dmb
1753  ret i32 %old
1754}
1755
1756define i64 @test_atomic_load_and_i64_release(i64 %offset) nounwind {
1757; CHECK-LABEL: test_atomic_load_and_i64_release:
1758  %old = atomicrmw and i64* @var64, i64 %offset release
1759; CHECK-NOT: dmb
1760; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]]
1761; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1762; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1763
1764; CHECK: ldclrl x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1765; CHECK-NOT: dmb
1766  ret i64 %old
1767}
1768
1769define void @test_atomic_load_and_i32_noret_release(i32 %offset) nounwind {
1770; CHECK-LABEL: test_atomic_load_and_i32_noret_release:
1771  atomicrmw and i32* @var32, i32 %offset release
1772; CHECK-NOT: dmb
1773; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1774; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1775; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1776
1777; CHECK: stclrl w[[NEW:[0-9]+]], [x[[ADDR]]]
1778; CHECK-NOT: dmb
1779  ret void
1780}
1781
1782define void @test_atomic_load_and_i64_noret_release(i64 %offset) nounwind {
1783; CHECK-LABEL: test_atomic_load_and_i64_noret_release:
1784  atomicrmw and i64* @var64, i64 %offset release
1785; CHECK-NOT: dmb
1786; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]]
1787; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1788; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1789
1790; CHECK: stclrl x[[NEW:[0-9]+]], [x[[ADDR]]]
1791; CHECK-NOT: dmb
1792  ret void
1793}
1794
1795define i8 @test_atomic_load_and_i8_seq_cst(i8 %offset) nounwind {
1796; CHECK-LABEL: test_atomic_load_and_i8_seq_cst:
1797  %old = atomicrmw and i8* @var8, i8 %offset seq_cst
1798; CHECK-NOT: dmb
1799; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1800; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1801; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1802
1803; CHECK: ldclralb w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1804; CHECK-NOT: dmb
1805  ret i8 %old
1806}
1807
1808define i16 @test_atomic_load_and_i16_seq_cst(i16 %offset) nounwind {
1809; CHECK-LABEL: test_atomic_load_and_i16_seq_cst:
1810  %old = atomicrmw and i16* @var16, i16 %offset seq_cst
1811; CHECK-NOT: dmb
1812; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1813; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1814; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1815
1816; CHECK: ldclralh w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1817; CHECK-NOT: dmb
1818  ret i16 %old
1819}
1820
1821define i32 @test_atomic_load_and_i32_seq_cst(i32 %offset) nounwind {
1822; CHECK-LABEL: test_atomic_load_and_i32_seq_cst:
1823  %old = atomicrmw and i32* @var32, i32 %offset seq_cst
1824; CHECK-NOT: dmb
1825; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1826; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1827; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1828
1829; CHECK: ldclral w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1830; CHECK-NOT: dmb
1831  ret i32 %old
1832}
1833
1834define i64 @test_atomic_load_and_i64_seq_cst(i64 %offset) nounwind {
1835; CHECK-LABEL: test_atomic_load_and_i64_seq_cst:
1836  %old = atomicrmw and i64* @var64, i64 %offset seq_cst
1837; CHECK-NOT: dmb
1838; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]]
1839; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1840; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1841
1842; CHECK: ldclral x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1843; CHECK-NOT: dmb
1844  ret i64 %old
1845}
1846
1847define void @test_atomic_load_and_i32_noret_seq_cst(i32 %offset) nounwind {
1848; CHECK-LABEL: test_atomic_load_and_i32_noret_seq_cst:
1849  atomicrmw and i32* @var32, i32 %offset seq_cst
1850; CHECK-NOT: dmb
1851; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]]
1852; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1853; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1854
1855; CHECK: ldclral w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]]
1856; CHECK-NOT: dmb
1857  ret void
1858}
1859
1860define void @test_atomic_load_and_i64_noret_seq_cst(i64 %offset) nounwind {
1861; CHECK-LABEL: test_atomic_load_and_i64_noret_seq_cst:
1862  atomicrmw and i64* @var64, i64 %offset seq_cst
1863; CHECK-NOT: dmb
1864; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]]
1865; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1866; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1867
1868; CHECK: ldclral x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]]
1869; CHECK-NOT: dmb
1870  ret void
1871}
1872
1873define i8 @test_atomic_cmpxchg_i8_acquire(i8 %wanted, i8 %new) nounwind {
1874; CHECK-LABEL: test_atomic_cmpxchg_i8_acquire:
1875   %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
1876   %old = extractvalue { i8, i1 } %pair, 0
1877
1878; CHECK-NOT: dmb
1879; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1880; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1881
1882; CHECK: casab w[[NEW:[0-9]+]], w[[OLD:[0-9]+]], [x[[ADDR]]]
1883; CHECK-NOT: dmb
1884
1885   ret i8 %old
1886}
1887
1888define i16 @test_atomic_cmpxchg_i16_acquire(i16 %wanted, i16 %new) nounwind {
1889; CHECK-LABEL: test_atomic_cmpxchg_i16_acquire:
1890   %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new acquire acquire
1891   %old = extractvalue { i16, i1 } %pair, 0
1892
1893; CHECK-NOT: dmb
1894; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1895; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1896
1897; CHECK: casah w0, w1, [x[[ADDR]]]
1898; CHECK-NOT: dmb
1899
1900   ret i16 %old
1901}
1902
1903define i32 @test_atomic_cmpxchg_i32_acquire(i32 %wanted, i32 %new) nounwind {
1904; CHECK-LABEL: test_atomic_cmpxchg_i32_acquire:
1905   %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new acquire acquire
1906   %old = extractvalue { i32, i1 } %pair, 0
1907
1908; CHECK-NOT: dmb
1909; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1910; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1911
1912; CHECK: casa w0, w1, [x[[ADDR]]]
1913; CHECK-NOT: dmb
1914
1915   ret i32 %old
1916}
1917
1918define i64 @test_atomic_cmpxchg_i64_acquire(i64 %wanted, i64 %new) nounwind {
1919; CHECK-LABEL: test_atomic_cmpxchg_i64_acquire:
1920   %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new acquire acquire
1921   %old = extractvalue { i64, i1 } %pair, 0
1922
1923; CHECK-NOT: dmb
1924; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1925; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1926
1927; CHECK: casa x0, x1, [x[[ADDR]]]
1928; CHECK-NOT: dmb
1929
1930   ret i64 %old
1931}
1932
1933define i128 @test_atomic_cmpxchg_i128_acquire(i128 %wanted, i128 %new) nounwind {
1934; CHECK-LABEL: test_atomic_cmpxchg_i128_acquire:
1935   %pair = cmpxchg i128* @var128, i128 %wanted, i128 %new acquire acquire
1936   %old = extractvalue { i128, i1 } %pair, 0
1937
1938; CHECK-NOT: dmb
1939; CHECK: adrp [[TMPADDR:x[0-9]+]], var128
1940; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128
1941
1942; CHECK: caspa x0, x1, x2, x3, [x[[ADDR]]]
1943; CHECK-NOT: dmb
1944
1945   ret i128 %old
1946}
1947
1948define i8 @test_atomic_cmpxchg_i8_monotonic(i8 %wanted, i8 %new) nounwind {
1949; CHECK-LABEL: test_atomic_cmpxchg_i8_monotonic:
1950   %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new monotonic monotonic
1951   %old = extractvalue { i8, i1 } %pair, 0
1952
1953; CHECK-NOT: dmb
1954; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1955; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1956
1957; CHECK: casb w[[NEW:[0-9]+]], w[[OLD:[0-9]+]], [x[[ADDR]]]
1958; CHECK-NOT: dmb
1959
1960   ret i8 %old
1961}
1962
1963define i16 @test_atomic_cmpxchg_i16_monotonic(i16 %wanted, i16 %new) nounwind {
1964; CHECK-LABEL: test_atomic_cmpxchg_i16_monotonic:
1965   %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new monotonic monotonic
1966   %old = extractvalue { i16, i1 } %pair, 0
1967
1968; CHECK-NOT: dmb
1969; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1970; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1971
1972; CHECK: cash w0, w1, [x[[ADDR]]]
1973; CHECK-NOT: dmb
1974
1975   ret i16 %old
1976}
1977
1978define i32 @test_atomic_cmpxchg_i32_monotonic(i32 %wanted, i32 %new) nounwind {
1979; CHECK-LABEL: test_atomic_cmpxchg_i32_monotonic:
1980   %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new monotonic monotonic
1981   %old = extractvalue { i32, i1 } %pair, 0
1982
1983; CHECK-NOT: dmb
1984; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1985; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1986
1987; CHECK: cas w0, w1, [x[[ADDR]]]
1988; CHECK-NOT: dmb
1989
1990   ret i32 %old
1991}
1992
1993define i64 @test_atomic_cmpxchg_i64_monotonic(i64 %wanted, i64 %new) nounwind {
1994; CHECK-LABEL: test_atomic_cmpxchg_i64_monotonic:
1995   %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic
1996   %old = extractvalue { i64, i1 } %pair, 0
1997
1998; CHECK-NOT: dmb
1999; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2000; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2001
2002; CHECK: cas x0, x1, [x[[ADDR]]]
2003; CHECK-NOT: dmb
2004
2005   ret i64 %old
2006}
2007
2008define i128 @test_atomic_cmpxchg_i128_monotonic(i128 %wanted, i128 %new) nounwind {
2009; CHECK-LABEL: test_atomic_cmpxchg_i128_monotonic:
2010   %pair = cmpxchg i128* @var128, i128 %wanted, i128 %new monotonic monotonic
2011   %old = extractvalue { i128, i1 } %pair, 0
2012
2013; CHECK-NOT: dmb
2014; CHECK: adrp [[TMPADDR:x[0-9]+]], var128
2015; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128
2016
2017; CHECK: casp x0, x1, x2, x3, [x[[ADDR]]]
2018; CHECK-NOT: dmb
2019
2020   ret i128 %old
2021}
2022
2023define i8 @test_atomic_cmpxchg_i8_seq_cst(i8 %wanted, i8 %new) nounwind {
2024; CHECK-LABEL: test_atomic_cmpxchg_i8_seq_cst:
2025   %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new seq_cst seq_cst
2026   %old = extractvalue { i8, i1 } %pair, 0
2027
2028; CHECK-NOT: dmb
2029; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
2030; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
2031
2032; CHECK: casalb w[[NEW:[0-9]+]], w[[OLD:[0-9]+]], [x[[ADDR]]]
2033; CHECK-NOT: dmb
2034
2035   ret i8 %old
2036}
2037
2038define i16 @test_atomic_cmpxchg_i16_seq_cst(i16 %wanted, i16 %new) nounwind {
2039; CHECK-LABEL: test_atomic_cmpxchg_i16_seq_cst:
2040   %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst
2041   %old = extractvalue { i16, i1 } %pair, 0
2042
2043; CHECK-NOT: dmb
2044; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
2045; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
2046
2047; CHECK: casalh w0, w1, [x[[ADDR]]]
2048; CHECK-NOT: dmb
2049
2050   ret i16 %old
2051}
2052
2053define i32 @test_atomic_cmpxchg_i32_seq_cst(i32 %wanted, i32 %new) nounwind {
2054; CHECK-LABEL: test_atomic_cmpxchg_i32_seq_cst:
2055   %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new seq_cst seq_cst
2056   %old = extractvalue { i32, i1 } %pair, 0
2057
2058; CHECK-NOT: dmb
2059; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2060; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2061
2062; CHECK: casal w0, w1, [x[[ADDR]]]
2063; CHECK-NOT: dmb
2064
2065   ret i32 %old
2066}
2067
2068define i64 @test_atomic_cmpxchg_i64_seq_cst(i64 %wanted, i64 %new) nounwind {
2069; CHECK-LABEL: test_atomic_cmpxchg_i64_seq_cst:
2070   %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new seq_cst seq_cst
2071   %old = extractvalue { i64, i1 } %pair, 0
2072
2073; CHECK-NOT: dmb
2074; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2075; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2076
2077; CHECK: casal x0, x1, [x[[ADDR]]]
2078; CHECK-NOT: dmb
2079
2080   ret i64 %old
2081}
2082
2083define i128 @test_atomic_cmpxchg_i128_seq_cst(i128 %wanted, i128 %new) nounwind {
2084; CHECK-LABEL: test_atomic_cmpxchg_i128_seq_cst:
2085   %pair = cmpxchg i128* @var128, i128 %wanted, i128 %new seq_cst seq_cst
2086   %old = extractvalue { i128, i1 } %pair, 0
2087
2088; CHECK-NOT: dmb
2089; CHECK: adrp [[TMPADDR:x[0-9]+]], var128
2090; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128
2091
2092; CHECK: caspal x0, x1, x2, x3, [x[[ADDR]]]
2093; CHECK-NOT: dmb
2094
2095   ret i128 %old
2096}
2097
2098define i8 @test_atomic_load_max_i8_acq_rel(i8 %offset) nounwind {
2099; CHECK-LABEL: test_atomic_load_max_i8_acq_rel:
2100   %old = atomicrmw max i8* @var8, i8 %offset acq_rel
2101; CHECK-NOT: dmb
2102; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
2103; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
2104
2105; CHECK: ldsmaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2106; CHECK-NOT: dmb
2107
2108   ret i8 %old
2109}
2110
2111define i16 @test_atomic_load_max_i16_acq_rel(i16 %offset) nounwind {
2112; CHECK-LABEL: test_atomic_load_max_i16_acq_rel:
2113   %old = atomicrmw max i16* @var16, i16 %offset acq_rel
2114; CHECK-NOT: dmb
2115; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
2116; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
2117
2118; CHECK: ldsmaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2119; CHECK-NOT: dmb
2120
2121   ret i16 %old
2122}
2123
2124define i32 @test_atomic_load_max_i32_acq_rel(i32 %offset) nounwind {
2125; CHECK-LABEL: test_atomic_load_max_i32_acq_rel:
2126   %old = atomicrmw max i32* @var32, i32 %offset acq_rel
2127; CHECK-NOT: dmb
2128; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2129; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2130
2131; CHECK: ldsmaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2132; CHECK-NOT: dmb
2133
2134   ret i32 %old
2135}
2136
2137define i64 @test_atomic_load_max_i64_acq_rel(i64 %offset) nounwind {
2138; CHECK-LABEL: test_atomic_load_max_i64_acq_rel:
2139   %old = atomicrmw max i64* @var64, i64 %offset acq_rel
2140; CHECK-NOT: dmb
2141; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2142; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2143
2144; CHECK: ldsmaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
2145; CHECK-NOT: dmb
2146
2147   ret i64 %old
2148}
2149
2150define void @test_atomic_load_max_i32_noret_acq_rel(i32 %offset) nounwind {
2151; CHECK-LABEL: test_atomic_load_max_i32_noret_acq_rel:
2152   atomicrmw max i32* @var32, i32 %offset acq_rel
2153; CHECK-NOT: dmb
2154; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2155; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2156
2157; CHECK: ldsmaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
2158; CHECK-NOT: dmb
2159  ret void
2160}
2161
2162define void @test_atomic_load_max_i64_noret_acq_rel(i64 %offset) nounwind {
2163; CHECK-LABEL: test_atomic_load_max_i64_noret_acq_rel:
2164   atomicrmw max i64* @var64, i64 %offset acq_rel
2165; CHECK-NOT: dmb
2166; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2167; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2168
2169; CHECK: ldsmaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
2170; CHECK-NOT: dmb
2171  ret void
2172}
2173
2174define i8 @test_atomic_load_max_i8_acquire(i8 %offset) nounwind {
2175; CHECK-LABEL: test_atomic_load_max_i8_acquire:
2176   %old = atomicrmw max i8* @var8, i8 %offset acquire
2177; CHECK-NOT: dmb
2178; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
2179; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
2180
2181; CHECK: ldsmaxab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2182; CHECK-NOT: dmb
2183
2184   ret i8 %old
2185}
2186
2187define i16 @test_atomic_load_max_i16_acquire(i16 %offset) nounwind {
2188; CHECK-LABEL: test_atomic_load_max_i16_acquire:
2189   %old = atomicrmw max i16* @var16, i16 %offset acquire
2190; CHECK-NOT: dmb
2191; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
2192; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
2193
2194; CHECK: ldsmaxah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2195; CHECK-NOT: dmb
2196
2197   ret i16 %old
2198}
2199
2200define i32 @test_atomic_load_max_i32_acquire(i32 %offset) nounwind {
2201; CHECK-LABEL: test_atomic_load_max_i32_acquire:
2202   %old = atomicrmw max i32* @var32, i32 %offset acquire
2203; CHECK-NOT: dmb
2204; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2205; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2206
2207; CHECK: ldsmaxa w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2208; CHECK-NOT: dmb
2209
2210   ret i32 %old
2211}
2212
2213define i64 @test_atomic_load_max_i64_acquire(i64 %offset) nounwind {
2214; CHECK-LABEL: test_atomic_load_max_i64_acquire:
2215   %old = atomicrmw max i64* @var64, i64 %offset acquire
2216; CHECK-NOT: dmb
2217; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2218; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2219
2220; CHECK: ldsmaxa x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
2221; CHECK-NOT: dmb
2222
2223   ret i64 %old
2224}
2225
2226define void @test_atomic_load_max_i32_noret_acquire(i32 %offset) nounwind {
2227; CHECK-LABEL: test_atomic_load_max_i32_noret_acquire:
2228   atomicrmw max i32* @var32, i32 %offset acquire
2229; CHECK-NOT: dmb
2230; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2231; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2232
2233; CHECK: ldsmaxa w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
2234; CHECK-NOT: dmb
2235  ret void
2236}
2237
2238define void @test_atomic_load_max_i64_noret_acquire(i64 %offset) nounwind {
2239; CHECK-LABEL: test_atomic_load_max_i64_noret_acquire:
2240   atomicrmw max i64* @var64, i64 %offset acquire
2241; CHECK-NOT: dmb
2242; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2243; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2244
2245; CHECK: ldsmaxa x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
2246; CHECK-NOT: dmb
2247  ret void
2248}
2249
2250define i8 @test_atomic_load_max_i8_monotonic(i8 %offset) nounwind {
2251; CHECK-LABEL: test_atomic_load_max_i8_monotonic:
2252   %old = atomicrmw max i8* @var8, i8 %offset monotonic
2253; CHECK-NOT: dmb
2254; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
2255; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
2256
2257; CHECK: ldsmaxb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2258; CHECK-NOT: dmb
2259
2260   ret i8 %old
2261}
2262
2263define i16 @test_atomic_load_max_i16_monotonic(i16 %offset) nounwind {
2264; CHECK-LABEL: test_atomic_load_max_i16_monotonic:
2265   %old = atomicrmw max i16* @var16, i16 %offset monotonic
2266; CHECK-NOT: dmb
2267; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
2268; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
2269
2270; CHECK: ldsmaxh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2271; CHECK-NOT: dmb
2272
2273   ret i16 %old
2274}
2275
2276define i32 @test_atomic_load_max_i32_monotonic(i32 %offset) nounwind {
2277; CHECK-LABEL: test_atomic_load_max_i32_monotonic:
2278   %old = atomicrmw max i32* @var32, i32 %offset monotonic
2279; CHECK-NOT: dmb
2280; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2281; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2282
2283; CHECK: ldsmax w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2284; CHECK-NOT: dmb
2285
2286   ret i32 %old
2287}
2288
2289define i64 @test_atomic_load_max_i64_monotonic(i64 %offset) nounwind {
2290; CHECK-LABEL: test_atomic_load_max_i64_monotonic:
2291   %old = atomicrmw max i64* @var64, i64 %offset monotonic
2292; CHECK-NOT: dmb
2293; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2294; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2295
2296; CHECK: ldsmax x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
2297; CHECK-NOT: dmb
2298
2299   ret i64 %old
2300}
2301
2302define void @test_atomic_load_max_i32_noret_monotonic(i32 %offset) nounwind {
2303; CHECK-LABEL: test_atomic_load_max_i32_noret_monotonic:
2304   atomicrmw max i32* @var32, i32 %offset monotonic
2305; CHECK-NOT: dmb
2306; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2307; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2308
2309; CHECK: stsmax w0, [x[[ADDR]]]
2310; CHECK-NOT: dmb
2311  ret void
2312}
2313
2314define void @test_atomic_load_max_i64_noret_monotonic(i64 %offset) nounwind {
2315; CHECK-LABEL: test_atomic_load_max_i64_noret_monotonic:
2316   atomicrmw max i64* @var64, i64 %offset monotonic
2317; CHECK-NOT: dmb
2318; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2319; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2320
2321; CHECK: stsmax x0, [x[[ADDR]]]
2322; CHECK-NOT: dmb
2323  ret void
2324}
2325
2326define i8 @test_atomic_load_max_i8_release(i8 %offset) nounwind {
2327; CHECK-LABEL: test_atomic_load_max_i8_release:
2328   %old = atomicrmw max i8* @var8, i8 %offset release
2329; CHECK-NOT: dmb
2330; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
2331; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
2332
2333; CHECK: ldsmaxlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2334; CHECK-NOT: dmb
2335
2336   ret i8 %old
2337}
2338
2339define i16 @test_atomic_load_max_i16_release(i16 %offset) nounwind {
2340; CHECK-LABEL: test_atomic_load_max_i16_release:
2341   %old = atomicrmw max i16* @var16, i16 %offset release
2342; CHECK-NOT: dmb
2343; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
2344; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
2345
2346; CHECK: ldsmaxlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2347; CHECK-NOT: dmb
2348
2349   ret i16 %old
2350}
2351
2352define i32 @test_atomic_load_max_i32_release(i32 %offset) nounwind {
2353; CHECK-LABEL: test_atomic_load_max_i32_release:
2354   %old = atomicrmw max i32* @var32, i32 %offset release
2355; CHECK-NOT: dmb
2356; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2357; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2358
2359; CHECK: ldsmaxl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2360; CHECK-NOT: dmb
2361
2362   ret i32 %old
2363}
2364
2365define i64 @test_atomic_load_max_i64_release(i64 %offset) nounwind {
2366; CHECK-LABEL: test_atomic_load_max_i64_release:
2367   %old = atomicrmw max i64* @var64, i64 %offset release
2368; CHECK-NOT: dmb
2369; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2370; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2371
2372; CHECK: ldsmaxl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
2373; CHECK-NOT: dmb
2374
2375   ret i64 %old
2376}
2377
2378define void @test_atomic_load_max_i32_noret_release(i32 %offset) nounwind {
2379; CHECK-LABEL: test_atomic_load_max_i32_noret_release:
2380   atomicrmw max i32* @var32, i32 %offset release
2381; CHECK-NOT: dmb
2382; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2383; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2384
2385; CHECK: stsmaxl w0, [x[[ADDR]]]
2386; CHECK-NOT: dmb
2387  ret void
2388}
2389
2390define void @test_atomic_load_max_i64_noret_release(i64 %offset) nounwind {
2391; CHECK-LABEL: test_atomic_load_max_i64_noret_release:
2392   atomicrmw max i64* @var64, i64 %offset release
2393; CHECK-NOT: dmb
2394; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2395; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2396
2397; CHECK: stsmaxl x0, [x[[ADDR]]]
2398; CHECK-NOT: dmb
2399  ret void
2400}
2401
2402define i8 @test_atomic_load_max_i8_seq_cst(i8 %offset) nounwind {
2403; CHECK-LABEL: test_atomic_load_max_i8_seq_cst:
2404   %old = atomicrmw max i8* @var8, i8 %offset seq_cst
2405; CHECK-NOT: dmb
2406; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
2407; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
2408
2409; CHECK: ldsmaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2410; CHECK-NOT: dmb
2411
2412   ret i8 %old
2413}
2414
2415define i16 @test_atomic_load_max_i16_seq_cst(i16 %offset) nounwind {
2416; CHECK-LABEL: test_atomic_load_max_i16_seq_cst:
2417   %old = atomicrmw max i16* @var16, i16 %offset seq_cst
2418; CHECK-NOT: dmb
2419; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
2420; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
2421
2422; CHECK: ldsmaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2423; CHECK-NOT: dmb
2424
2425   ret i16 %old
2426}
2427
2428define i32 @test_atomic_load_max_i32_seq_cst(i32 %offset) nounwind {
2429; CHECK-LABEL: test_atomic_load_max_i32_seq_cst:
2430   %old = atomicrmw max i32* @var32, i32 %offset seq_cst
2431; CHECK-NOT: dmb
2432; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2433; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2434
2435; CHECK: ldsmaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2436; CHECK-NOT: dmb
2437
2438   ret i32 %old
2439}
2440
2441define i64 @test_atomic_load_max_i64_seq_cst(i64 %offset) nounwind {
2442; CHECK-LABEL: test_atomic_load_max_i64_seq_cst:
2443   %old = atomicrmw max i64* @var64, i64 %offset seq_cst
2444; CHECK-NOT: dmb
2445; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2446; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2447
2448; CHECK: ldsmaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
2449; CHECK-NOT: dmb
2450
2451   ret i64 %old
2452}
2453
2454define void @test_atomic_load_max_i32_noret_seq_cst(i32 %offset) nounwind {
2455; CHECK-LABEL: test_atomic_load_max_i32_noret_seq_cst:
2456   atomicrmw max i32* @var32, i32 %offset seq_cst
2457; CHECK-NOT: dmb
2458; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2459; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2460
2461; CHECK: ldsmaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
2462; CHECK-NOT: dmb
2463  ret void
2464}
2465
2466define void @test_atomic_load_max_i64_noret_seq_cst(i64 %offset) nounwind {
2467; CHECK-LABEL: test_atomic_load_max_i64_noret_seq_cst:
2468   atomicrmw max i64* @var64, i64 %offset seq_cst
2469; CHECK-NOT: dmb
2470; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2471; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2472
2473; CHECK: ldsmaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
2474; CHECK-NOT: dmb
2475  ret void
2476}
2477
2478define i8 @test_atomic_load_min_i8_acq_rel(i8 %offset) nounwind {
2479; CHECK-LABEL: test_atomic_load_min_i8_acq_rel:
2480   %old = atomicrmw min i8* @var8, i8 %offset acq_rel
2481; CHECK-NOT: dmb
2482; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
2483; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
2484
2485; CHECK: ldsminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2486; CHECK-NOT: dmb
2487
2488   ret i8 %old
2489}
2490
2491define i16 @test_atomic_load_min_i16_acq_rel(i16 %offset) nounwind {
2492; CHECK-LABEL: test_atomic_load_min_i16_acq_rel:
2493   %old = atomicrmw min i16* @var16, i16 %offset acq_rel
2494; CHECK-NOT: dmb
2495; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
2496; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
2497
2498; CHECK: ldsminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2499; CHECK-NOT: dmb
2500
2501   ret i16 %old
2502}
2503
2504define i32 @test_atomic_load_min_i32_acq_rel(i32 %offset) nounwind {
2505; CHECK-LABEL: test_atomic_load_min_i32_acq_rel:
2506   %old = atomicrmw min i32* @var32, i32 %offset acq_rel
2507; CHECK-NOT: dmb
2508; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2509; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2510
2511; CHECK: ldsminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2512; CHECK-NOT: dmb
2513
2514   ret i32 %old
2515}
2516
2517define i64 @test_atomic_load_min_i64_acq_rel(i64 %offset) nounwind {
2518; CHECK-LABEL: test_atomic_load_min_i64_acq_rel:
2519   %old = atomicrmw min i64* @var64, i64 %offset acq_rel
2520; CHECK-NOT: dmb
2521; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2522; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2523
2524; CHECK: ldsminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
2525; CHECK-NOT: dmb
2526
2527   ret i64 %old
2528}
2529
2530define void @test_atomic_load_min_i32_noret_acq_rel(i32 %offset) nounwind {
2531; CHECK-LABEL: test_atomic_load_min_i32_noret_acq_rel:
2532   atomicrmw min i32* @var32, i32 %offset acq_rel
2533; CHECK-NOT: dmb
2534; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2535; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2536
2537; CHECK: ldsminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
2538; CHECK-NOT: dmb
2539  ret void
2540}
2541
2542define void @test_atomic_load_min_i64_noret_acq_rel(i64 %offset) nounwind {
2543; CHECK-LABEL: test_atomic_load_min_i64_noret_acq_rel:
2544   atomicrmw min i64* @var64, i64 %offset acq_rel
2545; CHECK-NOT: dmb
2546; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2547; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2548
2549; CHECK: ldsminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
2550; CHECK-NOT: dmb
2551  ret void
2552}
2553
2554define i8 @test_atomic_load_min_i8_acquire(i8 %offset) nounwind {
2555; CHECK-LABEL: test_atomic_load_min_i8_acquire:
2556   %old = atomicrmw min i8* @var8, i8 %offset acquire
2557; CHECK-NOT: dmb
2558; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
2559; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
2560
2561; CHECK: ldsminab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2562; CHECK-NOT: dmb
2563
2564   ret i8 %old
2565}
2566
2567define i16 @test_atomic_load_min_i16_acquire(i16 %offset) nounwind {
2568; CHECK-LABEL: test_atomic_load_min_i16_acquire:
2569   %old = atomicrmw min i16* @var16, i16 %offset acquire
2570; CHECK-NOT: dmb
2571; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
2572; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
2573
2574; CHECK: ldsminah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2575; CHECK-NOT: dmb
2576
2577   ret i16 %old
2578}
2579
2580define i32 @test_atomic_load_min_i32_acquire(i32 %offset) nounwind {
2581; CHECK-LABEL: test_atomic_load_min_i32_acquire:
2582   %old = atomicrmw min i32* @var32, i32 %offset acquire
2583; CHECK-NOT: dmb
2584; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2585; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2586
2587; CHECK: ldsmina w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2588; CHECK-NOT: dmb
2589
2590   ret i32 %old
2591}
2592
2593define i64 @test_atomic_load_min_i64_acquire(i64 %offset) nounwind {
2594; CHECK-LABEL: test_atomic_load_min_i64_acquire:
2595   %old = atomicrmw min i64* @var64, i64 %offset acquire
2596; CHECK-NOT: dmb
2597; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2598; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2599
2600; CHECK: ldsmina x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
2601; CHECK-NOT: dmb
2602
2603   ret i64 %old
2604}
2605
2606define void @test_atomic_load_min_i32_noret_acquire(i32 %offset) nounwind {
2607; CHECK-LABEL: test_atomic_load_min_i32_noret_acquire:
2608   atomicrmw min i32* @var32, i32 %offset acquire
2609; CHECK-NOT: dmb
2610; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2611; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2612
2613; CHECK: ldsmina w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
2614; CHECK-NOT: dmb
2615  ret void
2616}
2617
2618define void @test_atomic_load_min_i64_noret_acquire(i64 %offset) nounwind {
2619; CHECK-LABEL: test_atomic_load_min_i64_noret_acquire:
2620   atomicrmw min i64* @var64, i64 %offset acquire
2621; CHECK-NOT: dmb
2622; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2623; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2624
2625; CHECK: ldsmina x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
2626; CHECK-NOT: dmb
2627  ret void
2628}
2629
2630define i8 @test_atomic_load_min_i8_monotonic(i8 %offset) nounwind {
2631; CHECK-LABEL: test_atomic_load_min_i8_monotonic:
2632   %old = atomicrmw min i8* @var8, i8 %offset monotonic
2633; CHECK-NOT: dmb
2634; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
2635; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
2636
2637; CHECK: ldsminb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2638; CHECK-NOT: dmb
2639
2640   ret i8 %old
2641}
2642
2643define i16 @test_atomic_load_min_i16_monotonic(i16 %offset) nounwind {
2644; CHECK-LABEL: test_atomic_load_min_i16_monotonic:
2645   %old = atomicrmw min i16* @var16, i16 %offset monotonic
2646; CHECK-NOT: dmb
2647; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
2648; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
2649
2650; CHECK: ldsminh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2651; CHECK-NOT: dmb
2652
2653   ret i16 %old
2654}
2655
2656define i32 @test_atomic_load_min_i32_monotonic(i32 %offset) nounwind {
2657; CHECK-LABEL: test_atomic_load_min_i32_monotonic:
2658   %old = atomicrmw min i32* @var32, i32 %offset monotonic
2659; CHECK-NOT: dmb
2660; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2661; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2662
2663; CHECK: ldsmin w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2664; CHECK-NOT: dmb
2665
2666   ret i32 %old
2667}
2668
2669define i64 @test_atomic_load_min_i64_monotonic(i64 %offset) nounwind {
2670; CHECK-LABEL: test_atomic_load_min_i64_monotonic:
2671   %old = atomicrmw min i64* @var64, i64 %offset monotonic
2672; CHECK-NOT: dmb
2673; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2674; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2675
2676; CHECK: ldsmin x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
2677; CHECK-NOT: dmb
2678
2679   ret i64 %old
2680}
2681
2682define void @test_atomic_load_min_i32_noret_monotonic(i32 %offset) nounwind {
2683; CHECK-LABEL: test_atomic_load_min_i32_noret_monotonic:
2684   atomicrmw min i32* @var32, i32 %offset monotonic
2685; CHECK-NOT: dmb
2686; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2687; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2688
2689; CHECK: stsmin w0, [x[[ADDR]]]
2690; CHECK-NOT: dmb
2691  ret void
2692}
2693
2694define void @test_atomic_load_min_i64_noret_monotonic(i64 %offset) nounwind {
2695; CHECK-LABEL: test_atomic_load_min_i64_noret_monotonic:
2696   atomicrmw min i64* @var64, i64 %offset monotonic
2697; CHECK-NOT: dmb
2698; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2699; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2700
2701; CHECK: stsmin x0, [x[[ADDR]]]
2702; CHECK-NOT: dmb
2703  ret void
2704}
2705
2706define i8 @test_atomic_load_min_i8_release(i8 %offset) nounwind {
2707; CHECK-LABEL: test_atomic_load_min_i8_release:
2708   %old = atomicrmw min i8* @var8, i8 %offset release
2709; CHECK-NOT: dmb
2710; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
2711; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
2712
2713; CHECK: ldsminlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2714; CHECK-NOT: dmb
2715
2716   ret i8 %old
2717}
2718
2719define i16 @test_atomic_load_min_i16_release(i16 %offset) nounwind {
2720; CHECK-LABEL: test_atomic_load_min_i16_release:
2721   %old = atomicrmw min i16* @var16, i16 %offset release
2722; CHECK-NOT: dmb
2723; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
2724; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
2725
2726; CHECK: ldsminlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2727; CHECK-NOT: dmb
2728
2729   ret i16 %old
2730}
2731
2732define i32 @test_atomic_load_min_i32_release(i32 %offset) nounwind {
2733; CHECK-LABEL: test_atomic_load_min_i32_release:
2734   %old = atomicrmw min i32* @var32, i32 %offset release
2735; CHECK-NOT: dmb
2736; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2737; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2738
2739; CHECK: ldsminl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2740; CHECK-NOT: dmb
2741
2742   ret i32 %old
2743}
2744
2745define i64 @test_atomic_load_min_i64_release(i64 %offset) nounwind {
2746; CHECK-LABEL: test_atomic_load_min_i64_release:
2747   %old = atomicrmw min i64* @var64, i64 %offset release
2748; CHECK-NOT: dmb
2749; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2750; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2751
2752; CHECK: ldsminl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
2753; CHECK-NOT: dmb
2754
2755   ret i64 %old
2756}
2757
2758define void @test_atomic_load_min_i32_noret_release(i32 %offset) nounwind {
2759; CHECK-LABEL: test_atomic_load_min_i32_noret_release:
2760   atomicrmw min i32* @var32, i32 %offset release
2761; CHECK-NOT: dmb
2762; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2763; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2764
2765; CHECK: stsminl w0, [x[[ADDR]]]
2766; CHECK-NOT: dmb
2767  ret void
2768}
2769
2770define void @test_atomic_load_min_i64_noret_release(i64 %offset) nounwind {
2771; CHECK-LABEL: test_atomic_load_min_i64_noret_release:
2772   atomicrmw min i64* @var64, i64 %offset release
2773; CHECK-NOT: dmb
2774; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2775; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2776
2777; CHECK: stsminl x0, [x[[ADDR]]]
2778; CHECK-NOT: dmb
2779  ret void
2780}
2781
2782define i8 @test_atomic_load_min_i8_seq_cst(i8 %offset) nounwind {
2783; CHECK-LABEL: test_atomic_load_min_i8_seq_cst:
2784   %old = atomicrmw min i8* @var8, i8 %offset seq_cst
2785; CHECK-NOT: dmb
2786; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
2787; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
2788
2789; CHECK: ldsminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2790; CHECK-NOT: dmb
2791
2792   ret i8 %old
2793}
2794
2795define i16 @test_atomic_load_min_i16_seq_cst(i16 %offset) nounwind {
2796; CHECK-LABEL: test_atomic_load_min_i16_seq_cst:
2797   %old = atomicrmw min i16* @var16, i16 %offset seq_cst
2798; CHECK-NOT: dmb
2799; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
2800; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
2801
2802; CHECK: ldsminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2803; CHECK-NOT: dmb
2804
2805   ret i16 %old
2806}
2807
2808define i32 @test_atomic_load_min_i32_seq_cst(i32 %offset) nounwind {
2809; CHECK-LABEL: test_atomic_load_min_i32_seq_cst:
2810   %old = atomicrmw min i32* @var32, i32 %offset seq_cst
2811; CHECK-NOT: dmb
2812; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2813; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2814
2815; CHECK: ldsminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2816; CHECK-NOT: dmb
2817
2818   ret i32 %old
2819}
2820
2821define i64 @test_atomic_load_min_i64_seq_cst(i64 %offset) nounwind {
2822; CHECK-LABEL: test_atomic_load_min_i64_seq_cst:
2823   %old = atomicrmw min i64* @var64, i64 %offset seq_cst
2824; CHECK-NOT: dmb
2825; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2826; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2827
2828; CHECK: ldsminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
2829; CHECK-NOT: dmb
2830
2831   ret i64 %old
2832}
2833
2834define void @test_atomic_load_min_i32_noret_seq_cst(i32 %offset) nounwind {
2835; CHECK-LABEL: test_atomic_load_min_i32_noret_seq_cst:
2836   atomicrmw min i32* @var32, i32 %offset seq_cst
2837; CHECK-NOT: dmb
2838; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2839; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2840
2841; CHECK: ldsminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
2842; CHECK-NOT: dmb
2843  ret void
2844}
2845
2846define void @test_atomic_load_min_i64_noret_seq_cst(i64 %offset) nounwind {
2847; CHECK-LABEL: test_atomic_load_min_i64_noret_seq_cst:
2848   atomicrmw min i64* @var64, i64 %offset seq_cst
2849; CHECK-NOT: dmb
2850; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2851; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2852
2853; CHECK: ldsminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
2854; CHECK-NOT: dmb
2855  ret void
2856}
2857
2858define i8 @test_atomic_load_or_i8_acq_rel(i8 %offset) nounwind {
2859; CHECK-LABEL: test_atomic_load_or_i8_acq_rel:
2860   %old = atomicrmw or i8* @var8, i8 %offset acq_rel
2861; CHECK-NOT: dmb
2862; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
2863; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
2864
2865; CHECK: ldsetalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2866; CHECK-NOT: dmb
2867
2868   ret i8 %old
2869}
2870
2871define i16 @test_atomic_load_or_i16_acq_rel(i16 %offset) nounwind {
2872; CHECK-LABEL: test_atomic_load_or_i16_acq_rel:
2873   %old = atomicrmw or i16* @var16, i16 %offset acq_rel
2874; CHECK-NOT: dmb
2875; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
2876; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
2877
2878; CHECK: ldsetalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2879; CHECK-NOT: dmb
2880
2881   ret i16 %old
2882}
2883
2884define i32 @test_atomic_load_or_i32_acq_rel(i32 %offset) nounwind {
2885; CHECK-LABEL: test_atomic_load_or_i32_acq_rel:
2886   %old = atomicrmw or i32* @var32, i32 %offset acq_rel
2887; CHECK-NOT: dmb
2888; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2889; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2890
2891; CHECK: ldsetal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2892; CHECK-NOT: dmb
2893
2894   ret i32 %old
2895}
2896
2897define i64 @test_atomic_load_or_i64_acq_rel(i64 %offset) nounwind {
2898; CHECK-LABEL: test_atomic_load_or_i64_acq_rel:
2899   %old = atomicrmw or i64* @var64, i64 %offset acq_rel
2900; CHECK-NOT: dmb
2901; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2902; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2903
2904; CHECK: ldsetal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
2905; CHECK-NOT: dmb
2906
2907   ret i64 %old
2908}
2909
2910define void @test_atomic_load_or_i32_noret_acq_rel(i32 %offset) nounwind {
2911; CHECK-LABEL: test_atomic_load_or_i32_noret_acq_rel:
2912   atomicrmw or i32* @var32, i32 %offset acq_rel
2913; CHECK-NOT: dmb
2914; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2915; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2916
2917; CHECK: ldsetal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
2918; CHECK-NOT: dmb
2919  ret void
2920}
2921
2922define void @test_atomic_load_or_i64_noret_acq_rel(i64 %offset) nounwind {
2923; CHECK-LABEL: test_atomic_load_or_i64_noret_acq_rel:
2924   atomicrmw or i64* @var64, i64 %offset acq_rel
2925; CHECK-NOT: dmb
2926; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2927; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2928
2929; CHECK: ldsetal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
2930; CHECK-NOT: dmb
2931  ret void
2932}
2933
2934define i8 @test_atomic_load_or_i8_acquire(i8 %offset) nounwind {
2935; CHECK-LABEL: test_atomic_load_or_i8_acquire:
2936   %old = atomicrmw or i8* @var8, i8 %offset acquire
2937; CHECK-NOT: dmb
2938; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
2939; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
2940
2941; CHECK: ldsetab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2942; CHECK-NOT: dmb
2943
2944   ret i8 %old
2945}
2946
2947define i16 @test_atomic_load_or_i16_acquire(i16 %offset) nounwind {
2948; CHECK-LABEL: test_atomic_load_or_i16_acquire:
2949   %old = atomicrmw or i16* @var16, i16 %offset acquire
2950; CHECK-NOT: dmb
2951; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
2952; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
2953
2954; CHECK: ldsetah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2955; CHECK-NOT: dmb
2956
2957   ret i16 %old
2958}
2959
2960define i32 @test_atomic_load_or_i32_acquire(i32 %offset) nounwind {
2961; CHECK-LABEL: test_atomic_load_or_i32_acquire:
2962   %old = atomicrmw or i32* @var32, i32 %offset acquire
2963; CHECK-NOT: dmb
2964; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2965; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2966
2967; CHECK: ldseta w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
2968; CHECK-NOT: dmb
2969
2970   ret i32 %old
2971}
2972
2973define i64 @test_atomic_load_or_i64_acquire(i64 %offset) nounwind {
2974; CHECK-LABEL: test_atomic_load_or_i64_acquire:
2975   %old = atomicrmw or i64* @var64, i64 %offset acquire
2976; CHECK-NOT: dmb
2977; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
2978; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
2979
2980; CHECK: ldseta x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
2981; CHECK-NOT: dmb
2982
2983   ret i64 %old
2984}
2985
2986define void @test_atomic_load_or_i32_noret_acquire(i32 %offset) nounwind {
2987; CHECK-LABEL: test_atomic_load_or_i32_noret_acquire:
2988   atomicrmw or i32* @var32, i32 %offset acquire
2989; CHECK-NOT: dmb
2990; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
2991; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
2992
2993; CHECK: ldseta w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
2994; CHECK-NOT: dmb
2995  ret void
2996}
2997
2998define void @test_atomic_load_or_i64_noret_acquire(i64 %offset) nounwind {
2999; CHECK-LABEL: test_atomic_load_or_i64_noret_acquire:
3000   atomicrmw or i64* @var64, i64 %offset acquire
3001; CHECK-NOT: dmb
3002; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3003; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3004
3005; CHECK: ldseta x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
3006; CHECK-NOT: dmb
3007  ret void
3008}
3009
3010define i8 @test_atomic_load_or_i8_monotonic(i8 %offset) nounwind {
3011; CHECK-LABEL: test_atomic_load_or_i8_monotonic:
3012   %old = atomicrmw or i8* @var8, i8 %offset monotonic
3013; CHECK-NOT: dmb
3014; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
3015; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
3016
3017; CHECK: ldsetb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3018; CHECK-NOT: dmb
3019
3020   ret i8 %old
3021}
3022
3023define i16 @test_atomic_load_or_i16_monotonic(i16 %offset) nounwind {
3024; CHECK-LABEL: test_atomic_load_or_i16_monotonic:
3025   %old = atomicrmw or i16* @var16, i16 %offset monotonic
3026; CHECK-NOT: dmb
3027; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
3028; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
3029
3030; CHECK: ldseth w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3031; CHECK-NOT: dmb
3032
3033   ret i16 %old
3034}
3035
3036define i32 @test_atomic_load_or_i32_monotonic(i32 %offset) nounwind {
3037; CHECK-LABEL: test_atomic_load_or_i32_monotonic:
3038   %old = atomicrmw or i32* @var32, i32 %offset monotonic
3039; CHECK-NOT: dmb
3040; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3041; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3042
3043; CHECK: ldset w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3044; CHECK-NOT: dmb
3045
3046   ret i32 %old
3047}
3048
3049define i64 @test_atomic_load_or_i64_monotonic(i64 %offset) nounwind {
3050; CHECK-LABEL: test_atomic_load_or_i64_monotonic:
3051   %old = atomicrmw or i64* @var64, i64 %offset monotonic
3052; CHECK-NOT: dmb
3053; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3054; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3055
3056; CHECK: ldset x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3057; CHECK-NOT: dmb
3058
3059   ret i64 %old
3060}
3061
3062define void @test_atomic_load_or_i32_noret_monotonic(i32 %offset) nounwind {
3063; CHECK-LABEL: test_atomic_load_or_i32_noret_monotonic:
3064   atomicrmw or i32* @var32, i32 %offset monotonic
3065; CHECK-NOT: dmb
3066; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3067; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3068
3069; CHECK: stset w0, [x[[ADDR]]]
3070; CHECK-NOT: dmb
3071  ret void
3072}
3073
3074define void @test_atomic_load_or_i64_noret_monotonic(i64 %offset) nounwind {
3075; CHECK-LABEL: test_atomic_load_or_i64_noret_monotonic:
3076   atomicrmw or i64* @var64, i64 %offset monotonic
3077; CHECK-NOT: dmb
3078; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3079; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3080
3081; CHECK: stset x0, [x[[ADDR]]]
3082; CHECK-NOT: dmb
3083  ret void
3084}
3085
3086define i8 @test_atomic_load_or_i8_release(i8 %offset) nounwind {
3087; CHECK-LABEL: test_atomic_load_or_i8_release:
3088   %old = atomicrmw or i8* @var8, i8 %offset release
3089; CHECK-NOT: dmb
3090; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
3091; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
3092
3093; CHECK: ldsetlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3094; CHECK-NOT: dmb
3095
3096   ret i8 %old
3097}
3098
3099define i16 @test_atomic_load_or_i16_release(i16 %offset) nounwind {
3100; CHECK-LABEL: test_atomic_load_or_i16_release:
3101   %old = atomicrmw or i16* @var16, i16 %offset release
3102; CHECK-NOT: dmb
3103; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
3104; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
3105
3106; CHECK: ldsetlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3107; CHECK-NOT: dmb
3108
3109   ret i16 %old
3110}
3111
3112define i32 @test_atomic_load_or_i32_release(i32 %offset) nounwind {
3113; CHECK-LABEL: test_atomic_load_or_i32_release:
3114   %old = atomicrmw or i32* @var32, i32 %offset release
3115; CHECK-NOT: dmb
3116; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3117; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3118
3119; CHECK: ldsetl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3120; CHECK-NOT: dmb
3121
3122   ret i32 %old
3123}
3124
3125define i64 @test_atomic_load_or_i64_release(i64 %offset) nounwind {
3126; CHECK-LABEL: test_atomic_load_or_i64_release:
3127   %old = atomicrmw or i64* @var64, i64 %offset release
3128; CHECK-NOT: dmb
3129; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3130; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3131
3132; CHECK: ldsetl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3133; CHECK-NOT: dmb
3134
3135   ret i64 %old
3136}
3137
3138define void @test_atomic_load_or_i32_noret_release(i32 %offset) nounwind {
3139; CHECK-LABEL: test_atomic_load_or_i32_noret_release:
3140   atomicrmw or i32* @var32, i32 %offset release
3141; CHECK-NOT: dmb
3142; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3143; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3144
3145; CHECK: stsetl w0, [x[[ADDR]]]
3146; CHECK-NOT: dmb
3147  ret void
3148}
3149
3150define void @test_atomic_load_or_i64_noret_release(i64 %offset) nounwind {
3151; CHECK-LABEL: test_atomic_load_or_i64_noret_release:
3152   atomicrmw or i64* @var64, i64 %offset release
3153; CHECK-NOT: dmb
3154; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3155; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3156
3157; CHECK: stsetl x0, [x[[ADDR]]]
3158; CHECK-NOT: dmb
3159  ret void
3160}
3161
3162define i8 @test_atomic_load_or_i8_seq_cst(i8 %offset) nounwind {
3163; CHECK-LABEL: test_atomic_load_or_i8_seq_cst:
3164   %old = atomicrmw or i8* @var8, i8 %offset seq_cst
3165; CHECK-NOT: dmb
3166; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
3167; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
3168
3169; CHECK: ldsetalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3170; CHECK-NOT: dmb
3171
3172   ret i8 %old
3173}
3174
3175define i16 @test_atomic_load_or_i16_seq_cst(i16 %offset) nounwind {
3176; CHECK-LABEL: test_atomic_load_or_i16_seq_cst:
3177   %old = atomicrmw or i16* @var16, i16 %offset seq_cst
3178; CHECK-NOT: dmb
3179; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
3180; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
3181
3182; CHECK: ldsetalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3183; CHECK-NOT: dmb
3184
3185   ret i16 %old
3186}
3187
3188define i32 @test_atomic_load_or_i32_seq_cst(i32 %offset) nounwind {
3189; CHECK-LABEL: test_atomic_load_or_i32_seq_cst:
3190   %old = atomicrmw or i32* @var32, i32 %offset seq_cst
3191; CHECK-NOT: dmb
3192; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3193; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3194
3195; CHECK: ldsetal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3196; CHECK-NOT: dmb
3197
3198   ret i32 %old
3199}
3200
3201define i64 @test_atomic_load_or_i64_seq_cst(i64 %offset) nounwind {
3202; CHECK-LABEL: test_atomic_load_or_i64_seq_cst:
3203   %old = atomicrmw or i64* @var64, i64 %offset seq_cst
3204; CHECK-NOT: dmb
3205; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3206; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3207
3208; CHECK: ldsetal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3209; CHECK-NOT: dmb
3210
3211   ret i64 %old
3212}
3213
3214define void @test_atomic_load_or_i32_noret_seq_cst(i32 %offset) nounwind {
3215; CHECK-LABEL: test_atomic_load_or_i32_noret_seq_cst:
3216   atomicrmw or i32* @var32, i32 %offset seq_cst
3217; CHECK-NOT: dmb
3218; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3219; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3220
3221; CHECK: ldsetal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
3222; CHECK-NOT: dmb
3223  ret void
3224}
3225
3226define void @test_atomic_load_or_i64_noret_seq_cst(i64 %offset) nounwind {
3227; CHECK-LABEL: test_atomic_load_or_i64_noret_seq_cst:
3228   atomicrmw or i64* @var64, i64 %offset seq_cst
3229; CHECK-NOT: dmb
3230; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3231; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3232
3233; CHECK: ldsetal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
3234; CHECK-NOT: dmb
3235  ret void
3236}
3237
3238define i8 @test_atomic_load_sub_i8_acq_rel(i8 %offset) nounwind {
3239; CHECK-LABEL: test_atomic_load_sub_i8_acq_rel:
3240  %old = atomicrmw sub i8* @var8, i8 %offset acq_rel
3241; CHECK-NOT: dmb
3242; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3243; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
3244; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
3245
3246; CHECK: ldaddalb w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3247; CHECK-NOT: dmb
3248
3249  ret i8 %old
3250}
3251
3252define i16 @test_atomic_load_sub_i16_acq_rel(i16 %offset) nounwind {
3253; CHECK-LABEL: test_atomic_load_sub_i16_acq_rel:
3254  %old = atomicrmw sub i16* @var16, i16 %offset acq_rel
3255; CHECK-NOT: dmb
3256; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3257; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
3258; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
3259
3260; CHECK: ldaddalh w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3261; CHECK-NOT: dmb
3262
3263  ret i16 %old
3264}
3265
3266define i32 @test_atomic_load_sub_i32_acq_rel(i32 %offset) nounwind {
3267; CHECK-LABEL: test_atomic_load_sub_i32_acq_rel:
3268  %old = atomicrmw sub i32* @var32, i32 %offset acq_rel
3269; CHECK-NOT: dmb
3270; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3271; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3272; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3273
3274; CHECK: ldaddal w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3275; CHECK-NOT: dmb
3276
3277  ret i32 %old
3278}
3279
3280define i64 @test_atomic_load_sub_i64_acq_rel(i64 %offset) nounwind {
3281; CHECK-LABEL: test_atomic_load_sub_i64_acq_rel:
3282  %old = atomicrmw sub i64* @var64, i64 %offset acq_rel
3283; CHECK-NOT: dmb
3284; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]]
3285; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3286; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3287
3288; CHECK: ldaddal x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3289; CHECK-NOT: dmb
3290
3291  ret i64 %old
3292}
3293
3294define void @test_atomic_load_sub_i32_noret_acq_rel(i32 %offset) nounwind {
3295; CHECK-LABEL: test_atomic_load_sub_i32_noret_acq_rel:
3296  atomicrmw sub i32* @var32, i32 %offset acq_rel
3297; CHECK-NOT: dmb
3298; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3299; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3300; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3301
3302; CHECK: ldaddal w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3303; CHECK-NOT: dmb
3304
3305  ret void
3306}
3307
3308define void @test_atomic_load_sub_i64_noret_acq_rel(i64 %offset) nounwind {
3309; CHECK-LABEL: test_atomic_load_sub_i64_noret_acq_rel:
3310  atomicrmw sub i64* @var64, i64 %offset acq_rel
3311; CHECK-NOT: dmb
3312; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]]
3313; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3314; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3315
3316; CHECK: ldaddal x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3317; CHECK-NOT: dmb
3318
3319  ret void
3320}
3321
3322define i8 @test_atomic_load_sub_i8_acquire(i8 %offset) nounwind {
3323; CHECK-LABEL: test_atomic_load_sub_i8_acquire:
3324  %old = atomicrmw sub i8* @var8, i8 %offset acquire
3325; CHECK-NOT: dmb
3326; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3327; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
3328; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
3329
3330; CHECK: ldaddab w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3331; CHECK-NOT: dmb
3332
3333  ret i8 %old
3334}
3335
3336define i16 @test_atomic_load_sub_i16_acquire(i16 %offset) nounwind {
3337; CHECK-LABEL: test_atomic_load_sub_i16_acquire:
3338  %old = atomicrmw sub i16* @var16, i16 %offset acquire
3339; CHECK-NOT: dmb
3340; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3341; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
3342; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
3343
3344; CHECK: ldaddah w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3345; CHECK-NOT: dmb
3346
3347  ret i16 %old
3348}
3349
3350define i32 @test_atomic_load_sub_i32_acquire(i32 %offset) nounwind {
3351; CHECK-LABEL: test_atomic_load_sub_i32_acquire:
3352  %old = atomicrmw sub i32* @var32, i32 %offset acquire
3353; CHECK-NOT: dmb
3354; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3355; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3356; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3357
3358; CHECK: ldadda w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3359; CHECK-NOT: dmb
3360
3361  ret i32 %old
3362}
3363
3364define i64 @test_atomic_load_sub_i64_acquire(i64 %offset) nounwind {
3365; CHECK-LABEL: test_atomic_load_sub_i64_acquire:
3366  %old = atomicrmw sub i64* @var64, i64 %offset acquire
3367; CHECK-NOT: dmb
3368; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]]
3369; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3370; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3371
3372; CHECK: ldadda x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3373; CHECK-NOT: dmb
3374
3375  ret i64 %old
3376}
3377
3378define void @test_atomic_load_sub_i32_noret_acquire(i32 %offset) nounwind {
3379; CHECK-LABEL: test_atomic_load_sub_i32_noret_acquire:
3380  atomicrmw sub i32* @var32, i32 %offset acquire
3381; CHECK-NOT: dmb
3382; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3383; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3384; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3385
3386; CHECK: ldadda w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3387; CHECK-NOT: dmb
3388
3389  ret void
3390}
3391
3392define void @test_atomic_load_sub_i64_noret_acquire(i64 %offset) nounwind {
3393; CHECK-LABEL: test_atomic_load_sub_i64_noret_acquire:
3394  atomicrmw sub i64* @var64, i64 %offset acquire
3395; CHECK-NOT: dmb
3396; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]]
3397; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3398; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3399
3400; CHECK: ldadda x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3401; CHECK-NOT: dmb
3402
3403  ret void
3404}
3405
3406define i8 @test_atomic_load_sub_i8_monotonic(i8 %offset) nounwind {
3407; CHECK-LABEL: test_atomic_load_sub_i8_monotonic:
3408  %old = atomicrmw sub i8* @var8, i8 %offset monotonic
3409; CHECK-NOT: dmb
3410; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3411; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
3412; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
3413
3414; CHECK: ldaddb w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3415; CHECK-NOT: dmb
3416
3417  ret i8 %old
3418}
3419
3420define i16 @test_atomic_load_sub_i16_monotonic(i16 %offset) nounwind {
3421; CHECK-LABEL: test_atomic_load_sub_i16_monotonic:
3422  %old = atomicrmw sub i16* @var16, i16 %offset monotonic
3423; CHECK-NOT: dmb
3424; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3425; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
3426; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
3427
3428; CHECK: ldaddh w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3429; CHECK-NOT: dmb
3430
3431  ret i16 %old
3432}
3433
3434define i32 @test_atomic_load_sub_i32_monotonic(i32 %offset) nounwind {
3435; CHECK-LABEL: test_atomic_load_sub_i32_monotonic:
3436  %old = atomicrmw sub i32* @var32, i32 %offset monotonic
3437; CHECK-NOT: dmb
3438; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3439; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3440; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3441
3442; CHECK: ldadd w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3443; CHECK-NOT: dmb
3444
3445  ret i32 %old
3446}
3447
3448define i64 @test_atomic_load_sub_i64_monotonic(i64 %offset) nounwind {
3449; CHECK-LABEL: test_atomic_load_sub_i64_monotonic:
3450  %old = atomicrmw sub i64* @var64, i64 %offset monotonic
3451; CHECK-NOT: dmb
3452; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]]
3453; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3454; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3455
3456; CHECK: ldadd x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3457; CHECK-NOT: dmb
3458
3459  ret i64 %old
3460}
3461
3462define void @test_atomic_load_sub_i32_noret_monotonic(i32 %offset) nounwind {
3463; CHECK-LABEL: test_atomic_load_sub_i32_noret_monotonic:
3464  atomicrmw sub i32* @var32, i32 %offset monotonic
3465; CHECK-NOT: dmb
3466; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3467; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3468; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3469
3470; CHECK: stadd w[[NEW:[0-9]+]], [x[[ADDR]]]
3471; CHECK-NOT: dmb
3472
3473  ret void
3474}
3475
3476define void @test_atomic_load_sub_i64_noret_monotonic(i64 %offset) nounwind {
3477; CHECK-LABEL: test_atomic_load_sub_i64_noret_monotonic:
3478  atomicrmw sub i64* @var64, i64 %offset monotonic
3479; CHECK-NOT: dmb
3480; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]]
3481; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3482; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3483
3484; CHECK: stadd x[[NEW:[0-9]+]], [x[[ADDR]]]
3485; CHECK-NOT: dmb
3486
3487  ret void
3488}
3489
3490define i8 @test_atomic_load_sub_i8_release(i8 %offset) nounwind {
3491; CHECK-LABEL: test_atomic_load_sub_i8_release:
3492  %old = atomicrmw sub i8* @var8, i8 %offset release
3493; CHECK-NOT: dmb
3494; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3495; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
3496; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
3497
3498; CHECK: ldaddlb w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3499; CHECK-NOT: dmb
3500
3501  ret i8 %old
3502}
3503
3504define i16 @test_atomic_load_sub_i16_release(i16 %offset) nounwind {
3505; CHECK-LABEL: test_atomic_load_sub_i16_release:
3506  %old = atomicrmw sub i16* @var16, i16 %offset release
3507; CHECK-NOT: dmb
3508; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3509; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
3510; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
3511
3512; CHECK: ldaddlh w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3513; CHECK-NOT: dmb
3514
3515  ret i16 %old
3516}
3517
3518define i32 @test_atomic_load_sub_i32_release(i32 %offset) nounwind {
3519; CHECK-LABEL: test_atomic_load_sub_i32_release:
3520  %old = atomicrmw sub i32* @var32, i32 %offset release
3521; CHECK-NOT: dmb
3522; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3523; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3524; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3525
3526; CHECK: ldaddl w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3527; CHECK-NOT: dmb
3528
3529  ret i32 %old
3530}
3531
3532define i64 @test_atomic_load_sub_i64_release(i64 %offset) nounwind {
3533; CHECK-LABEL: test_atomic_load_sub_i64_release:
3534  %old = atomicrmw sub i64* @var64, i64 %offset release
3535; CHECK-NOT: dmb
3536; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]]
3537; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3538; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3539
3540; CHECK: ldaddl x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3541; CHECK-NOT: dmb
3542
3543  ret i64 %old
3544}
3545
3546define void @test_atomic_load_sub_i32_noret_release(i32 %offset) nounwind {
3547; CHECK-LABEL: test_atomic_load_sub_i32_noret_release:
3548  atomicrmw sub i32* @var32, i32 %offset release
3549; CHECK-NOT: dmb
3550; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3551; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3552; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3553
3554; CHECK: staddl w[[NEW:[0-9]+]], [x[[ADDR]]]
3555; CHECK-NOT: dmb
3556
3557  ret void
3558}
3559
3560define void @test_atomic_load_sub_i64_noret_release(i64 %offset) nounwind {
3561; CHECK-LABEL: test_atomic_load_sub_i64_noret_release:
3562  atomicrmw sub i64* @var64, i64 %offset release
3563; CHECK-NOT: dmb
3564; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]]
3565; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3566; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3567
3568; CHECK: staddl x[[NEW:[0-9]+]], [x[[ADDR]]]
3569; CHECK-NOT: dmb
3570
3571  ret void
3572}
3573
3574define i8 @test_atomic_load_sub_i8_seq_cst(i8 %offset) nounwind {
3575; CHECK-LABEL: test_atomic_load_sub_i8_seq_cst:
3576  %old = atomicrmw sub i8* @var8, i8 %offset seq_cst
3577; CHECK-NOT: dmb
3578; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3579; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
3580; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
3581
3582; CHECK: ldaddalb w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3583; CHECK-NOT: dmb
3584
3585  ret i8 %old
3586}
3587
3588define i16 @test_atomic_load_sub_i16_seq_cst(i16 %offset) nounwind {
3589; CHECK-LABEL: test_atomic_load_sub_i16_seq_cst:
3590  %old = atomicrmw sub i16* @var16, i16 %offset seq_cst
3591; CHECK-NOT: dmb
3592; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3593; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
3594; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
3595
3596; CHECK: ldaddalh w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3597; CHECK-NOT: dmb
3598
3599  ret i16 %old
3600}
3601
3602define i32 @test_atomic_load_sub_i32_seq_cst(i32 %offset) nounwind {
3603; CHECK-LABEL: test_atomic_load_sub_i32_seq_cst:
3604  %old = atomicrmw sub i32* @var32, i32 %offset seq_cst
3605; CHECK-NOT: dmb
3606; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3607; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3608; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3609
3610; CHECK: ldaddal w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3611; CHECK-NOT: dmb
3612
3613  ret i32 %old
3614}
3615
3616define i64 @test_atomic_load_sub_i64_seq_cst(i64 %offset) nounwind {
3617; CHECK-LABEL: test_atomic_load_sub_i64_seq_cst:
3618  %old = atomicrmw sub i64* @var64, i64 %offset seq_cst
3619; CHECK-NOT: dmb
3620; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]]
3621; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3622; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3623
3624; CHECK: ldaddal x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3625; CHECK-NOT: dmb
3626
3627  ret i64 %old
3628}
3629
3630define void @test_atomic_load_sub_i32_noret_seq_cst(i32 %offset) nounwind {
3631; CHECK-LABEL: test_atomic_load_sub_i32_noret_seq_cst:
3632  atomicrmw sub i32* @var32, i32 %offset seq_cst
3633; CHECK-NOT: dmb
3634; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]]
3635; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3636; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3637
3638; CHECK: ldaddal w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3639; CHECK-NOT: dmb
3640
3641  ret void
3642}
3643
3644define void @test_atomic_load_sub_i64_noret_seq_cst(i64 %offset) nounwind {
3645; CHECK-LABEL: test_atomic_load_sub_i64_noret_seq_cst:
3646  atomicrmw sub i64* @var64, i64 %offset seq_cst
3647; CHECK-NOT: dmb
3648; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]]
3649; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3650; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3651
3652; CHECK: ldaddal x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3653; CHECK-NOT: dmb
3654
3655  ret void
3656}
3657
3658define i8 @test_atomic_load_xchg_i8_acq_rel(i8 %offset) nounwind {
3659; CHECK-LABEL: test_atomic_load_xchg_i8_acq_rel:
3660   %old = atomicrmw xchg i8* @var8, i8 %offset acq_rel
3661; CHECK-NOT: dmb
3662; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
3663; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
3664
3665; CHECK: swpalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3666; CHECK-NOT: dmb
3667
3668   ret i8 %old
3669}
3670
3671define i16 @test_atomic_load_xchg_i16_acq_rel(i16 %offset) nounwind {
3672; CHECK-LABEL: test_atomic_load_xchg_i16_acq_rel:
3673   %old = atomicrmw xchg i16* @var16, i16 %offset acq_rel
3674; CHECK-NOT: dmb
3675; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
3676; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
3677
3678; CHECK: swpalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3679; CHECK-NOT: dmb
3680
3681   ret i16 %old
3682}
3683
3684define i32 @test_atomic_load_xchg_i32_acq_rel(i32 %offset) nounwind {
3685; CHECK-LABEL: test_atomic_load_xchg_i32_acq_rel:
3686   %old = atomicrmw xchg i32* @var32, i32 %offset acq_rel
3687; CHECK-NOT: dmb
3688; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3689; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3690
3691; CHECK: swpal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3692; CHECK-NOT: dmb
3693
3694   ret i32 %old
3695}
3696
3697define i64 @test_atomic_load_xchg_i64_acq_rel(i64 %offset) nounwind {
3698; CHECK-LABEL: test_atomic_load_xchg_i64_acq_rel:
3699   %old = atomicrmw xchg i64* @var64, i64 %offset acq_rel
3700; CHECK-NOT: dmb
3701; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3702; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3703
3704; CHECK: swpal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3705; CHECK-NOT: dmb
3706
3707   ret i64 %old
3708}
3709
3710define void @test_atomic_load_xchg_i32_noret_acq_rel(i32 %offset) nounwind {
3711; CHECK-LABEL: test_atomic_load_xchg_i32_noret_acq_rel:
3712   atomicrmw xchg i32* @var32, i32 %offset acq_rel
3713; CHECK-NOT: dmb
3714; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3715; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3716
3717; CHECK: swpal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3718; CHECK-NOT: dmb
3719
3720   ret void
3721}
3722
3723define void @test_atomic_load_xchg_i64_noret_acq_rel(i64 %offset) nounwind {
3724; CHECK-LABEL: test_atomic_load_xchg_i64_noret_acq_rel:
3725   atomicrmw xchg i64* @var64, i64 %offset acq_rel
3726; CHECK-NOT: dmb
3727; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3728; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3729
3730; CHECK: swpal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3731; CHECK-NOT: dmb
3732
3733   ret void
3734}
3735
3736define i8 @test_atomic_load_xchg_i8_acquire(i8 %offset) nounwind {
3737; CHECK-LABEL: test_atomic_load_xchg_i8_acquire:
3738   %old = atomicrmw xchg i8* @var8, i8 %offset acquire
3739; CHECK-NOT: dmb
3740; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
3741; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
3742
3743; CHECK: swpab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3744; CHECK-NOT: dmb
3745
3746   ret i8 %old
3747}
3748
3749define i16 @test_atomic_load_xchg_i16_acquire(i16 %offset) nounwind {
3750; CHECK-LABEL: test_atomic_load_xchg_i16_acquire:
3751   %old = atomicrmw xchg i16* @var16, i16 %offset acquire
3752; CHECK-NOT: dmb
3753; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
3754; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
3755
3756; CHECK: swpah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3757; CHECK-NOT: dmb
3758
3759   ret i16 %old
3760}
3761
3762define i32 @test_atomic_load_xchg_i32_acquire(i32 %offset) nounwind {
3763; CHECK-LABEL: test_atomic_load_xchg_i32_acquire:
3764   %old = atomicrmw xchg i32* @var32, i32 %offset acquire
3765; CHECK-NOT: dmb
3766; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3767; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3768
3769; CHECK: swpa w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3770; CHECK-NOT: dmb
3771
3772   ret i32 %old
3773}
3774
3775define i64 @test_atomic_load_xchg_i64_acquire(i64 %offset) nounwind {
3776; CHECK-LABEL: test_atomic_load_xchg_i64_acquire:
3777   %old = atomicrmw xchg i64* @var64, i64 %offset acquire
3778; CHECK-NOT: dmb
3779; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3780; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3781
3782; CHECK: swpa x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3783; CHECK-NOT: dmb
3784
3785   ret i64 %old
3786}
3787
3788define void @test_atomic_load_xchg_i32_noret_acquire(i32 %offset) nounwind {
3789; CHECK-LABEL: test_atomic_load_xchg_i32_noret_acquire:
3790   atomicrmw xchg i32* @var32, i32 %offset acquire
3791; CHECK-NOT: dmb
3792; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3793; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3794
3795; CHECK: swpa w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3796; CHECK-NOT: dmb
3797
3798   ret void
3799}
3800
3801define void @test_atomic_load_xchg_i64_noret_acquire(i64 %offset) nounwind {
3802; CHECK-LABEL: test_atomic_load_xchg_i64_noret_acquire:
3803   atomicrmw xchg i64* @var64, i64 %offset acquire
3804; CHECK-NOT: dmb
3805; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3806; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3807
3808; CHECK: swpa x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3809; CHECK-NOT: dmb
3810
3811   ret void
3812}
3813
3814define i8 @test_atomic_load_xchg_i8_monotonic(i8 %offset) nounwind {
3815; CHECK-LABEL: test_atomic_load_xchg_i8_monotonic:
3816   %old = atomicrmw xchg i8* @var8, i8 %offset monotonic
3817; CHECK-NOT: dmb
3818; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
3819; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
3820
3821; CHECK: swpb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3822; CHECK-NOT: dmb
3823
3824   ret i8 %old
3825}
3826
3827define i16 @test_atomic_load_xchg_i16_monotonic(i16 %offset) nounwind {
3828; CHECK-LABEL: test_atomic_load_xchg_i16_monotonic:
3829   %old = atomicrmw xchg i16* @var16, i16 %offset monotonic
3830; CHECK-NOT: dmb
3831; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
3832; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
3833
3834; CHECK: swph w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3835; CHECK-NOT: dmb
3836
3837   ret i16 %old
3838}
3839
3840define i32 @test_atomic_load_xchg_i32_monotonic(i32 %offset) nounwind {
3841; CHECK-LABEL: test_atomic_load_xchg_i32_monotonic:
3842   %old = atomicrmw xchg i32* @var32, i32 %offset monotonic
3843; CHECK-NOT: dmb
3844; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3845; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3846
3847; CHECK: swp w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3848; CHECK-NOT: dmb
3849
3850   ret i32 %old
3851}
3852
3853define i64 @test_atomic_load_xchg_i64_monotonic(i64 %offset) nounwind {
3854; CHECK-LABEL: test_atomic_load_xchg_i64_monotonic:
3855   %old = atomicrmw xchg i64* @var64, i64 %offset monotonic
3856; CHECK-NOT: dmb
3857; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3858; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3859
3860; CHECK: swp x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3861; CHECK-NOT: dmb
3862
3863   ret i64 %old
3864}
3865
3866define void @test_atomic_load_xchg_i32_noret_monotonic(i32 %offset) nounwind {
3867; CHECK-LABEL: test_atomic_load_xchg_i32_noret_monotonic:
3868   atomicrmw xchg i32* @var32, i32 %offset monotonic
3869; CHECK-NOT: dmb
3870; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3871; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3872
3873; CHECK: swp w[[OLD:[0-9]+]], w[[NEW:[0-9,a-z]+]], [x[[ADDR]]]
3874; CHECK-NOT: dmb
3875
3876   ret void
3877}
3878
3879define void @test_atomic_load_xchg_i64_noret_monotonic(i64 %offset) nounwind {
3880; CHECK-LABEL: test_atomic_load_xchg_i64_noret_monotonic:
3881   atomicrmw xchg i64* @var64, i64 %offset monotonic
3882; CHECK-NOT: dmb
3883; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3884; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3885
3886; CHECK: swp x[[OLD:[0-9]+]], x[[NEW:[0-9,a-z]+]], [x[[ADDR]]]
3887; CHECK-NOT: dmb
3888
3889   ret void
3890}
3891
3892define i8 @test_atomic_load_xchg_i8_release(i8 %offset) nounwind {
3893; CHECK-LABEL: test_atomic_load_xchg_i8_release:
3894   %old = atomicrmw xchg i8* @var8, i8 %offset release
3895; CHECK-NOT: dmb
3896; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
3897; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
3898
3899; CHECK: swplb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3900; CHECK-NOT: dmb
3901
3902   ret i8 %old
3903}
3904
3905define i16 @test_atomic_load_xchg_i16_release(i16 %offset) nounwind {
3906; CHECK-LABEL: test_atomic_load_xchg_i16_release:
3907   %old = atomicrmw xchg i16* @var16, i16 %offset release
3908; CHECK-NOT: dmb
3909; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
3910; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
3911
3912; CHECK: swplh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3913; CHECK-NOT: dmb
3914
3915   ret i16 %old
3916}
3917
3918define i32 @test_atomic_load_xchg_i32_release(i32 %offset) nounwind {
3919; CHECK-LABEL: test_atomic_load_xchg_i32_release:
3920   %old = atomicrmw xchg i32* @var32, i32 %offset release
3921; CHECK-NOT: dmb
3922; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3923; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3924
3925; CHECK: swpl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3926; CHECK-NOT: dmb
3927
3928   ret i32 %old
3929}
3930
3931define i64 @test_atomic_load_xchg_i64_release(i64 %offset) nounwind {
3932; CHECK-LABEL: test_atomic_load_xchg_i64_release:
3933   %old = atomicrmw xchg i64* @var64, i64 %offset release
3934; CHECK-NOT: dmb
3935; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3936; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3937
3938; CHECK: swpl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
3939; CHECK-NOT: dmb
3940
3941   ret i64 %old
3942}
3943
3944define void @test_atomic_load_xchg_i32_noret_release(i32 %offset) nounwind {
3945; CHECK-LABEL: test_atomic_load_xchg_i32_noret_release:
3946   atomicrmw xchg i32* @var32, i32 %offset release
3947; CHECK-NOT: dmb
3948; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
3949; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
3950
3951; CHECK: swpl w[[OLD:[0-9]+]], w[[NEW:[0-9,a-z]+]], [x[[ADDR]]]
3952; CHECK-NOT: dmb
3953
3954   ret void
3955}
3956
3957define void @test_atomic_load_xchg_i64_noret_release(i64 %offset) nounwind {
3958; CHECK-LABEL: test_atomic_load_xchg_i64_noret_release:
3959   atomicrmw xchg i64* @var64, i64 %offset release
3960; CHECK-NOT: dmb
3961; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
3962; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
3963
3964; CHECK: swpl x[[OLD:[0-9]+]], x[[NEW:[0-9,a-z]+]], [x[[ADDR]]]
3965; CHECK-NOT: dmb
3966
3967   ret void
3968}
3969
3970define i8 @test_atomic_load_xchg_i8_seq_cst(i8 %offset) nounwind {
3971; CHECK-LABEL: test_atomic_load_xchg_i8_seq_cst:
3972   %old = atomicrmw xchg i8* @var8, i8 %offset seq_cst
3973; CHECK-NOT: dmb
3974; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
3975; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
3976
3977; CHECK: swpalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3978; CHECK-NOT: dmb
3979
3980   ret i8 %old
3981}
3982
3983define i16 @test_atomic_load_xchg_i16_seq_cst(i16 %offset) nounwind {
3984; CHECK-LABEL: test_atomic_load_xchg_i16_seq_cst:
3985   %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst
3986; CHECK-NOT: dmb
3987; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
3988; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
3989
3990; CHECK: swpalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
3991; CHECK-NOT: dmb
3992
3993   ret i16 %old
3994}
3995
3996define i32 @test_atomic_load_xchg_i32_seq_cst(i32 %offset) nounwind {
3997; CHECK-LABEL: test_atomic_load_xchg_i32_seq_cst:
3998   %old = atomicrmw xchg i32* @var32, i32 %offset seq_cst
3999; CHECK-NOT: dmb
4000; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4001; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4002
4003; CHECK: swpal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4004; CHECK-NOT: dmb
4005
4006   ret i32 %old
4007}
4008
4009define i64 @test_atomic_load_xchg_i64_seq_cst(i64 %offset) nounwind {
4010; CHECK-LABEL: test_atomic_load_xchg_i64_seq_cst:
4011   %old = atomicrmw xchg i64* @var64, i64 %offset seq_cst
4012; CHECK-NOT: dmb
4013; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4014; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4015
4016; CHECK: swpal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4017; CHECK-NOT: dmb
4018
4019   ret i64 %old
4020}
4021
4022define void @test_atomic_load_xchg_i32_noret_seq_cst(i32 %offset) nounwind {
4023; CHECK-LABEL: test_atomic_load_xchg_i32_noret_seq_cst:
4024   atomicrmw xchg i32* @var32, i32 %offset seq_cst
4025; CHECK-NOT: dmb
4026; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4027; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4028
4029; CHECK: swpal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4030; CHECK-NOT: dmb
4031
4032   ret void
4033}
4034
4035define void @test_atomic_load_xchg_i64_noret_seq_cst(i64 %offset) nounwind {
4036; CHECK-LABEL: test_atomic_load_xchg_i64_noret_seq_cst:
4037   atomicrmw xchg i64* @var64, i64 %offset seq_cst
4038; CHECK-NOT: dmb
4039; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4040; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4041
4042; CHECK: swpal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4043; CHECK-NOT: dmb
4044
4045   ret void
4046}
4047
4048define i8 @test_atomic_load_umax_i8_acq_rel(i8 %offset) nounwind {
4049; CHECK-LABEL: test_atomic_load_umax_i8_acq_rel:
4050   %old = atomicrmw umax i8* @var8, i8 %offset acq_rel
4051; CHECK-NOT: dmb
4052; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
4053; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
4054
4055; CHECK: ldumaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4056; CHECK-NOT: dmb
4057
4058   ret i8 %old
4059}
4060
4061define i16 @test_atomic_load_umax_i16_acq_rel(i16 %offset) nounwind {
4062; CHECK-LABEL: test_atomic_load_umax_i16_acq_rel:
4063   %old = atomicrmw umax i16* @var16, i16 %offset acq_rel
4064; CHECK-NOT: dmb
4065; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
4066; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
4067
4068; CHECK: ldumaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4069; CHECK-NOT: dmb
4070
4071   ret i16 %old
4072}
4073
4074define i32 @test_atomic_load_umax_i32_acq_rel(i32 %offset) nounwind {
4075; CHECK-LABEL: test_atomic_load_umax_i32_acq_rel:
4076   %old = atomicrmw umax i32* @var32, i32 %offset acq_rel
4077; CHECK-NOT: dmb
4078; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4079; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4080
4081; CHECK: ldumaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4082; CHECK-NOT: dmb
4083
4084   ret i32 %old
4085}
4086
4087define i64 @test_atomic_load_umax_i64_acq_rel(i64 %offset) nounwind {
4088; CHECK-LABEL: test_atomic_load_umax_i64_acq_rel:
4089   %old = atomicrmw umax i64* @var64, i64 %offset acq_rel
4090; CHECK-NOT: dmb
4091; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4092; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4093
4094; CHECK: ldumaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4095; CHECK-NOT: dmb
4096
4097   ret i64 %old
4098}
4099
4100define void @test_atomic_load_umax_i32_noret_acq_rel(i32 %offset) nounwind {
4101; CHECK-LABEL: test_atomic_load_umax_i32_noret_acq_rel:
4102   atomicrmw umax i32* @var32, i32 %offset acq_rel
4103; CHECK-NOT: dmb
4104; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4105; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4106
4107; CHECK: ldumaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
4108; CHECK-NOT: dmb
4109  ret void
4110}
4111
4112define void @test_atomic_load_umax_i64_noret_acq_rel(i64 %offset) nounwind {
4113; CHECK-LABEL: test_atomic_load_umax_i64_noret_acq_rel:
4114   atomicrmw umax i64* @var64, i64 %offset acq_rel
4115; CHECK-NOT: dmb
4116; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4117; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4118
4119; CHECK: ldumaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
4120; CHECK-NOT: dmb
4121  ret void
4122}
4123
4124define i8 @test_atomic_load_umax_i8_acquire(i8 %offset) nounwind {
4125; CHECK-LABEL: test_atomic_load_umax_i8_acquire:
4126   %old = atomicrmw umax i8* @var8, i8 %offset acquire
4127; CHECK-NOT: dmb
4128; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
4129; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
4130
4131; CHECK: ldumaxab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4132; CHECK-NOT: dmb
4133
4134   ret i8 %old
4135}
4136
4137define i16 @test_atomic_load_umax_i16_acquire(i16 %offset) nounwind {
4138; CHECK-LABEL: test_atomic_load_umax_i16_acquire:
4139   %old = atomicrmw umax i16* @var16, i16 %offset acquire
4140; CHECK-NOT: dmb
4141; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
4142; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
4143
4144; CHECK: ldumaxah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4145; CHECK-NOT: dmb
4146
4147   ret i16 %old
4148}
4149
4150define i32 @test_atomic_load_umax_i32_acquire(i32 %offset) nounwind {
4151; CHECK-LABEL: test_atomic_load_umax_i32_acquire:
4152   %old = atomicrmw umax i32* @var32, i32 %offset acquire
4153; CHECK-NOT: dmb
4154; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4155; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4156
4157; CHECK: ldumaxa w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4158; CHECK-NOT: dmb
4159
4160   ret i32 %old
4161}
4162
4163define i64 @test_atomic_load_umax_i64_acquire(i64 %offset) nounwind {
4164; CHECK-LABEL: test_atomic_load_umax_i64_acquire:
4165   %old = atomicrmw umax i64* @var64, i64 %offset acquire
4166; CHECK-NOT: dmb
4167; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4168; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4169
4170; CHECK: ldumaxa x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4171; CHECK-NOT: dmb
4172
4173   ret i64 %old
4174}
4175
4176define void @test_atomic_load_umax_i32_noret_acquire(i32 %offset) nounwind {
4177; CHECK-LABEL: test_atomic_load_umax_i32_noret_acquire:
4178   atomicrmw umax i32* @var32, i32 %offset acquire
4179; CHECK-NOT: dmb
4180; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4181; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4182
4183; CHECK: ldumaxa w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
4184; CHECK-NOT: dmb
4185  ret void
4186}
4187
4188define void @test_atomic_load_umax_i64_noret_acquire(i64 %offset) nounwind {
4189; CHECK-LABEL: test_atomic_load_umax_i64_noret_acquire:
4190   atomicrmw umax i64* @var64, i64 %offset acquire
4191; CHECK-NOT: dmb
4192; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4193; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4194
4195; CHECK: ldumaxa x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
4196; CHECK-NOT: dmb
4197  ret void
4198}
4199
4200define i8 @test_atomic_load_umax_i8_monotonic(i8 %offset) nounwind {
4201; CHECK-LABEL: test_atomic_load_umax_i8_monotonic:
4202   %old = atomicrmw umax i8* @var8, i8 %offset monotonic
4203; CHECK-NOT: dmb
4204; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
4205; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
4206
4207; CHECK: ldumaxb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4208; CHECK-NOT: dmb
4209
4210   ret i8 %old
4211}
4212
4213define i16 @test_atomic_load_umax_i16_monotonic(i16 %offset) nounwind {
4214; CHECK-LABEL: test_atomic_load_umax_i16_monotonic:
4215   %old = atomicrmw umax i16* @var16, i16 %offset monotonic
4216; CHECK-NOT: dmb
4217; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
4218; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
4219
4220; CHECK: ldumaxh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4221; CHECK-NOT: dmb
4222
4223   ret i16 %old
4224}
4225
4226define i32 @test_atomic_load_umax_i32_monotonic(i32 %offset) nounwind {
4227; CHECK-LABEL: test_atomic_load_umax_i32_monotonic:
4228   %old = atomicrmw umax i32* @var32, i32 %offset monotonic
4229; CHECK-NOT: dmb
4230; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4231; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4232
4233; CHECK: ldumax w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4234; CHECK-NOT: dmb
4235
4236   ret i32 %old
4237}
4238
4239define i64 @test_atomic_load_umax_i64_monotonic(i64 %offset) nounwind {
4240; CHECK-LABEL: test_atomic_load_umax_i64_monotonic:
4241   %old = atomicrmw umax i64* @var64, i64 %offset monotonic
4242; CHECK-NOT: dmb
4243; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4244; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4245
4246; CHECK: ldumax x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4247; CHECK-NOT: dmb
4248
4249   ret i64 %old
4250}
4251
4252define void @test_atomic_load_umax_i32_noret_monotonic(i32 %offset) nounwind {
4253; CHECK-LABEL: test_atomic_load_umax_i32_noret_monotonic:
4254   atomicrmw umax i32* @var32, i32 %offset monotonic
4255; CHECK-NOT: dmb
4256; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4257; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4258
4259; CHECK: stumax w0, [x[[ADDR]]]
4260; CHECK-NOT: dmb
4261  ret void
4262}
4263
4264define void @test_atomic_load_umax_i64_noret_monotonic(i64 %offset) nounwind {
4265; CHECK-LABEL: test_atomic_load_umax_i64_noret_monotonic:
4266   atomicrmw umax i64* @var64, i64 %offset monotonic
4267; CHECK-NOT: dmb
4268; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4269; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4270
4271; CHECK: stumax x0, [x[[ADDR]]]
4272; CHECK-NOT: dmb
4273  ret void
4274}
4275
4276define i8 @test_atomic_load_umax_i8_release(i8 %offset) nounwind {
4277; CHECK-LABEL: test_atomic_load_umax_i8_release:
4278   %old = atomicrmw umax i8* @var8, i8 %offset release
4279; CHECK-NOT: dmb
4280; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
4281; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
4282
4283; CHECK: ldumaxlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4284; CHECK-NOT: dmb
4285
4286   ret i8 %old
4287}
4288
4289define i16 @test_atomic_load_umax_i16_release(i16 %offset) nounwind {
4290; CHECK-LABEL: test_atomic_load_umax_i16_release:
4291   %old = atomicrmw umax i16* @var16, i16 %offset release
4292; CHECK-NOT: dmb
4293; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
4294; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
4295
4296; CHECK: ldumaxlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4297; CHECK-NOT: dmb
4298
4299   ret i16 %old
4300}
4301
4302define i32 @test_atomic_load_umax_i32_release(i32 %offset) nounwind {
4303; CHECK-LABEL: test_atomic_load_umax_i32_release:
4304   %old = atomicrmw umax i32* @var32, i32 %offset release
4305; CHECK-NOT: dmb
4306; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4307; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4308
4309; CHECK: ldumaxl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4310; CHECK-NOT: dmb
4311
4312   ret i32 %old
4313}
4314
4315define i64 @test_atomic_load_umax_i64_release(i64 %offset) nounwind {
4316; CHECK-LABEL: test_atomic_load_umax_i64_release:
4317   %old = atomicrmw umax i64* @var64, i64 %offset release
4318; CHECK-NOT: dmb
4319; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4320; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4321
4322; CHECK: ldumaxl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4323; CHECK-NOT: dmb
4324
4325   ret i64 %old
4326}
4327
4328define void @test_atomic_load_umax_i32_noret_release(i32 %offset) nounwind {
4329; CHECK-LABEL: test_atomic_load_umax_i32_noret_release:
4330   atomicrmw umax i32* @var32, i32 %offset release
4331; CHECK-NOT: dmb
4332; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4333; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4334
4335; CHECK: stumaxl w0, [x[[ADDR]]]
4336; CHECK-NOT: dmb
4337  ret void
4338}
4339
4340define void @test_atomic_load_umax_i64_noret_release(i64 %offset) nounwind {
4341; CHECK-LABEL: test_atomic_load_umax_i64_noret_release:
4342   atomicrmw umax i64* @var64, i64 %offset release
4343; CHECK-NOT: dmb
4344; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4345; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4346
4347; CHECK: stumaxl x0, [x[[ADDR]]]
4348; CHECK-NOT: dmb
4349  ret void
4350}
4351
4352define i8 @test_atomic_load_umax_i8_seq_cst(i8 %offset) nounwind {
4353; CHECK-LABEL: test_atomic_load_umax_i8_seq_cst:
4354   %old = atomicrmw umax i8* @var8, i8 %offset seq_cst
4355; CHECK-NOT: dmb
4356; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
4357; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
4358
4359; CHECK: ldumaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4360; CHECK-NOT: dmb
4361
4362   ret i8 %old
4363}
4364
4365define i16 @test_atomic_load_umax_i16_seq_cst(i16 %offset) nounwind {
4366; CHECK-LABEL: test_atomic_load_umax_i16_seq_cst:
4367   %old = atomicrmw umax i16* @var16, i16 %offset seq_cst
4368; CHECK-NOT: dmb
4369; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
4370; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
4371
4372; CHECK: ldumaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4373; CHECK-NOT: dmb
4374
4375   ret i16 %old
4376}
4377
4378define i32 @test_atomic_load_umax_i32_seq_cst(i32 %offset) nounwind {
4379; CHECK-LABEL: test_atomic_load_umax_i32_seq_cst:
4380   %old = atomicrmw umax i32* @var32, i32 %offset seq_cst
4381; CHECK-NOT: dmb
4382; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4383; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4384
4385; CHECK: ldumaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4386; CHECK-NOT: dmb
4387
4388   ret i32 %old
4389}
4390
4391define i64 @test_atomic_load_umax_i64_seq_cst(i64 %offset) nounwind {
4392; CHECK-LABEL: test_atomic_load_umax_i64_seq_cst:
4393   %old = atomicrmw umax i64* @var64, i64 %offset seq_cst
4394; CHECK-NOT: dmb
4395; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4396; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4397
4398; CHECK: ldumaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4399; CHECK-NOT: dmb
4400
4401   ret i64 %old
4402}
4403
4404define void @test_atomic_load_umax_i32_noret_seq_cst(i32 %offset) nounwind {
4405; CHECK-LABEL: test_atomic_load_umax_i32_noret_seq_cst:
4406   atomicrmw umax i32* @var32, i32 %offset seq_cst
4407; CHECK-NOT: dmb
4408; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4409; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4410
4411; CHECK: ldumaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
4412; CHECK-NOT: dmb
4413  ret void
4414}
4415
4416define void @test_atomic_load_umax_i64_noret_seq_cst(i64 %offset) nounwind {
4417; CHECK-LABEL: test_atomic_load_umax_i64_noret_seq_cst:
4418   atomicrmw umax i64* @var64, i64 %offset seq_cst
4419; CHECK-NOT: dmb
4420; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4421; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4422
4423; CHECK: ldumaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
4424; CHECK-NOT: dmb
4425  ret void
4426}
4427
4428define i8 @test_atomic_load_umin_i8_acq_rel(i8 %offset) nounwind {
4429; CHECK-LABEL: test_atomic_load_umin_i8_acq_rel:
4430   %old = atomicrmw umin i8* @var8, i8 %offset acq_rel
4431; CHECK-NOT: dmb
4432; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
4433; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
4434
4435; CHECK: lduminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4436; CHECK-NOT: dmb
4437
4438   ret i8 %old
4439}
4440
4441define i16 @test_atomic_load_umin_i16_acq_rel(i16 %offset) nounwind {
4442; CHECK-LABEL: test_atomic_load_umin_i16_acq_rel:
4443   %old = atomicrmw umin i16* @var16, i16 %offset acq_rel
4444; CHECK-NOT: dmb
4445; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
4446; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
4447
4448; CHECK: lduminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4449; CHECK-NOT: dmb
4450
4451   ret i16 %old
4452}
4453
4454define i32 @test_atomic_load_umin_i32_acq_rel(i32 %offset) nounwind {
4455; CHECK-LABEL: test_atomic_load_umin_i32_acq_rel:
4456   %old = atomicrmw umin i32* @var32, i32 %offset acq_rel
4457; CHECK-NOT: dmb
4458; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4459; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4460
4461; CHECK: lduminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4462; CHECK-NOT: dmb
4463
4464   ret i32 %old
4465}
4466
4467define i64 @test_atomic_load_umin_i64_acq_rel(i64 %offset) nounwind {
4468; CHECK-LABEL: test_atomic_load_umin_i64_acq_rel:
4469   %old = atomicrmw umin i64* @var64, i64 %offset acq_rel
4470; CHECK-NOT: dmb
4471; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4472; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4473
4474; CHECK: lduminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4475; CHECK-NOT: dmb
4476
4477   ret i64 %old
4478}
4479
4480define void @test_atomic_load_umin_i32_noret_acq_rel(i32 %offset) nounwind {
4481; CHECK-LABEL: test_atomic_load_umin_i32_noret_acq_rel:
4482   atomicrmw umin i32* @var32, i32 %offset acq_rel
4483; CHECK-NOT: dmb
4484; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4485; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4486
4487; CHECK: lduminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
4488; CHECK-NOT: dmb
4489  ret void
4490}
4491
4492define void @test_atomic_load_umin_i64_noret_acq_rel(i64 %offset) nounwind {
4493; CHECK-LABEL: test_atomic_load_umin_i64_noret_acq_rel:
4494   atomicrmw umin i64* @var64, i64 %offset acq_rel
4495; CHECK-NOT: dmb
4496; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4497; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4498
4499; CHECK: lduminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
4500; CHECK-NOT: dmb
4501  ret void
4502}
4503
4504define i8 @test_atomic_load_umin_i8_acquire(i8 %offset) nounwind {
4505; CHECK-LABEL: test_atomic_load_umin_i8_acquire:
4506   %old = atomicrmw umin i8* @var8, i8 %offset acquire
4507; CHECK-NOT: dmb
4508; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
4509; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
4510
4511; CHECK: lduminab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4512; CHECK-NOT: dmb
4513
4514   ret i8 %old
4515}
4516
4517define i16 @test_atomic_load_umin_i16_acquire(i16 %offset) nounwind {
4518; CHECK-LABEL: test_atomic_load_umin_i16_acquire:
4519   %old = atomicrmw umin i16* @var16, i16 %offset acquire
4520; CHECK-NOT: dmb
4521; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
4522; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
4523
4524; CHECK: lduminah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4525; CHECK-NOT: dmb
4526
4527   ret i16 %old
4528}
4529
4530define i32 @test_atomic_load_umin_i32_acquire(i32 %offset) nounwind {
4531; CHECK-LABEL: test_atomic_load_umin_i32_acquire:
4532   %old = atomicrmw umin i32* @var32, i32 %offset acquire
4533; CHECK-NOT: dmb
4534; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4535; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4536
4537; CHECK: ldumina w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4538; CHECK-NOT: dmb
4539
4540   ret i32 %old
4541}
4542
4543define i64 @test_atomic_load_umin_i64_acquire(i64 %offset) nounwind {
4544; CHECK-LABEL: test_atomic_load_umin_i64_acquire:
4545   %old = atomicrmw umin i64* @var64, i64 %offset acquire
4546; CHECK-NOT: dmb
4547; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4548; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4549
4550; CHECK: ldumina x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4551; CHECK-NOT: dmb
4552
4553   ret i64 %old
4554}
4555
4556define void @test_atomic_load_umin_i32_noret_acquire(i32 %offset) nounwind {
4557; CHECK-LABEL: test_atomic_load_umin_i32_noret_acquire:
4558   atomicrmw umin i32* @var32, i32 %offset acquire
4559; CHECK-NOT: dmb
4560; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4561; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4562
4563; CHECK: ldumina w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
4564; CHECK-NOT: dmb
4565  ret void
4566}
4567
4568define void @test_atomic_load_umin_i64_noret_acquire(i64 %offset) nounwind {
4569; CHECK-LABEL: test_atomic_load_umin_i64_noret_acquire:
4570   atomicrmw umin i64* @var64, i64 %offset acquire
4571; CHECK-NOT: dmb
4572; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4573; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4574
4575; CHECK: ldumina x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
4576; CHECK-NOT: dmb
4577  ret void
4578}
4579
4580define i8 @test_atomic_load_umin_i8_monotonic(i8 %offset) nounwind {
4581; CHECK-LABEL: test_atomic_load_umin_i8_monotonic:
4582   %old = atomicrmw umin i8* @var8, i8 %offset monotonic
4583; CHECK-NOT: dmb
4584; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
4585; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
4586
4587; CHECK: lduminb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4588; CHECK-NOT: dmb
4589
4590   ret i8 %old
4591}
4592
4593define i16 @test_atomic_load_umin_i16_monotonic(i16 %offset) nounwind {
4594; CHECK-LABEL: test_atomic_load_umin_i16_monotonic:
4595   %old = atomicrmw umin i16* @var16, i16 %offset monotonic
4596; CHECK-NOT: dmb
4597; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
4598; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
4599
4600; CHECK: lduminh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4601; CHECK-NOT: dmb
4602
4603   ret i16 %old
4604}
4605
4606define i32 @test_atomic_load_umin_i32_monotonic(i32 %offset) nounwind {
4607; CHECK-LABEL: test_atomic_load_umin_i32_monotonic:
4608   %old = atomicrmw umin i32* @var32, i32 %offset monotonic
4609; CHECK-NOT: dmb
4610; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4611; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4612
4613; CHECK: ldumin w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4614; CHECK-NOT: dmb
4615
4616   ret i32 %old
4617}
4618
4619define i64 @test_atomic_load_umin_i64_monotonic(i64 %offset) nounwind {
4620; CHECK-LABEL: test_atomic_load_umin_i64_monotonic:
4621   %old = atomicrmw umin i64* @var64, i64 %offset monotonic
4622; CHECK-NOT: dmb
4623; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4624; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4625
4626; CHECK: ldumin x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4627; CHECK-NOT: dmb
4628
4629   ret i64 %old
4630}
4631
4632define void @test_atomic_load_umin_i32_noret_monotonic(i32 %offset) nounwind {
4633; CHECK-LABEL: test_atomic_load_umin_i32_noret_monotonic:
4634   atomicrmw umin i32* @var32, i32 %offset monotonic
4635; CHECK-NOT: dmb
4636; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4637; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4638
4639; CHECK: stumin w0, [x[[ADDR]]]
4640; CHECK-NOT: dmb
4641  ret void
4642}
4643
4644define void @test_atomic_load_umin_i64_noret_monotonic(i64 %offset) nounwind {
4645; CHECK-LABEL: test_atomic_load_umin_i64_noret_monotonic:
4646   atomicrmw umin i64* @var64, i64 %offset monotonic
4647; CHECK-NOT: dmb
4648; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4649; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4650
4651; CHECK: stumin x0, [x[[ADDR]]]
4652; CHECK-NOT: dmb
4653  ret void
4654}
4655
4656define i8 @test_atomic_load_umin_i8_release(i8 %offset) nounwind {
4657; CHECK-LABEL: test_atomic_load_umin_i8_release:
4658   %old = atomicrmw umin i8* @var8, i8 %offset release
4659; CHECK-NOT: dmb
4660; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
4661; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
4662
4663; CHECK: lduminlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4664; CHECK-NOT: dmb
4665
4666   ret i8 %old
4667}
4668
4669define i16 @test_atomic_load_umin_i16_release(i16 %offset) nounwind {
4670; CHECK-LABEL: test_atomic_load_umin_i16_release:
4671   %old = atomicrmw umin i16* @var16, i16 %offset release
4672; CHECK-NOT: dmb
4673; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
4674; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
4675
4676; CHECK: lduminlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4677; CHECK-NOT: dmb
4678
4679   ret i16 %old
4680}
4681
4682define i32 @test_atomic_load_umin_i32_release(i32 %offset) nounwind {
4683; CHECK-LABEL: test_atomic_load_umin_i32_release:
4684   %old = atomicrmw umin i32* @var32, i32 %offset release
4685; CHECK-NOT: dmb
4686; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4687; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4688
4689; CHECK: lduminl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4690; CHECK-NOT: dmb
4691
4692   ret i32 %old
4693}
4694
4695define i64 @test_atomic_load_umin_i64_release(i64 %offset) nounwind {
4696; CHECK-LABEL: test_atomic_load_umin_i64_release:
4697   %old = atomicrmw umin i64* @var64, i64 %offset release
4698; CHECK-NOT: dmb
4699; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4700; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4701
4702; CHECK: lduminl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4703; CHECK-NOT: dmb
4704
4705   ret i64 %old
4706}
4707
4708define void @test_atomic_load_umin_i32_noret_release(i32 %offset) nounwind {
4709; CHECK-LABEL: test_atomic_load_umin_i32_noret_release:
4710   atomicrmw umin i32* @var32, i32 %offset release
4711; CHECK-NOT: dmb
4712; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4713; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4714
4715; CHECK: stuminl w0, [x[[ADDR]]]
4716; CHECK-NOT: dmb
4717  ret void
4718}
4719
4720define void @test_atomic_load_umin_i64_noret_release(i64 %offset) nounwind {
4721; CHECK-LABEL: test_atomic_load_umin_i64_noret_release:
4722   atomicrmw umin i64* @var64, i64 %offset release
4723; CHECK-NOT: dmb
4724; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4725; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4726
4727; CHECK: stuminl x0, [x[[ADDR]]]
4728; CHECK-NOT: dmb
4729  ret void
4730}
4731
4732define i8 @test_atomic_load_umin_i8_seq_cst(i8 %offset) nounwind {
4733; CHECK-LABEL: test_atomic_load_umin_i8_seq_cst:
4734   %old = atomicrmw umin i8* @var8, i8 %offset seq_cst
4735; CHECK-NOT: dmb
4736; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
4737; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
4738
4739; CHECK: lduminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4740; CHECK-NOT: dmb
4741
4742   ret i8 %old
4743}
4744
4745define i16 @test_atomic_load_umin_i16_seq_cst(i16 %offset) nounwind {
4746; CHECK-LABEL: test_atomic_load_umin_i16_seq_cst:
4747   %old = atomicrmw umin i16* @var16, i16 %offset seq_cst
4748; CHECK-NOT: dmb
4749; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
4750; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
4751
4752; CHECK: lduminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4753; CHECK-NOT: dmb
4754
4755   ret i16 %old
4756}
4757
4758define i32 @test_atomic_load_umin_i32_seq_cst(i32 %offset) nounwind {
4759; CHECK-LABEL: test_atomic_load_umin_i32_seq_cst:
4760   %old = atomicrmw umin i32* @var32, i32 %offset seq_cst
4761; CHECK-NOT: dmb
4762; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4763; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4764
4765; CHECK: lduminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4766; CHECK-NOT: dmb
4767
4768   ret i32 %old
4769}
4770
4771define i64 @test_atomic_load_umin_i64_seq_cst(i64 %offset) nounwind {
4772; CHECK-LABEL: test_atomic_load_umin_i64_seq_cst:
4773   %old = atomicrmw umin i64* @var64, i64 %offset seq_cst
4774; CHECK-NOT: dmb
4775; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4776; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4777
4778; CHECK: lduminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4779; CHECK-NOT: dmb
4780
4781   ret i64 %old
4782}
4783
4784define void @test_atomic_load_umin_i32_noret_seq_cst(i32 %offset) nounwind {
4785; CHECK-LABEL: test_atomic_load_umin_i32_noret_seq_cst:
4786   atomicrmw umin i32* @var32, i32 %offset seq_cst
4787; CHECK-NOT: dmb
4788; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4789; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4790
4791; CHECK: lduminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
4792; CHECK-NOT: dmb
4793  ret void
4794}
4795
4796define void @test_atomic_load_umin_i64_noret_seq_cst(i64 %offset) nounwind {
4797; CHECK-LABEL: test_atomic_load_umin_i64_noret_seq_cst:
4798   atomicrmw umin i64* @var64, i64 %offset seq_cst
4799; CHECK-NOT: dmb
4800; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4801; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4802
4803; CHECK: lduminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
4804; CHECK-NOT: dmb
4805  ret void
4806}
4807
4808define i8 @test_atomic_load_xor_i8_acq_rel(i8 %offset) nounwind {
4809; CHECK-LABEL: test_atomic_load_xor_i8_acq_rel:
4810   %old = atomicrmw xor i8* @var8, i8 %offset acq_rel
4811; CHECK-NOT: dmb
4812; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
4813; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
4814
4815; CHECK: ldeoralb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4816; CHECK-NOT: dmb
4817
4818   ret i8 %old
4819}
4820
4821define i16 @test_atomic_load_xor_i16_acq_rel(i16 %offset) nounwind {
4822; CHECK-LABEL: test_atomic_load_xor_i16_acq_rel:
4823   %old = atomicrmw xor i16* @var16, i16 %offset acq_rel
4824; CHECK-NOT: dmb
4825; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
4826; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
4827
4828; CHECK: ldeoralh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4829; CHECK-NOT: dmb
4830
4831   ret i16 %old
4832}
4833
4834define i32 @test_atomic_load_xor_i32_acq_rel(i32 %offset) nounwind {
4835; CHECK-LABEL: test_atomic_load_xor_i32_acq_rel:
4836   %old = atomicrmw xor i32* @var32, i32 %offset acq_rel
4837; CHECK-NOT: dmb
4838; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4839; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4840
4841; CHECK: ldeoral w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4842; CHECK-NOT: dmb
4843
4844   ret i32 %old
4845}
4846
4847define i64 @test_atomic_load_xor_i64_acq_rel(i64 %offset) nounwind {
4848; CHECK-LABEL: test_atomic_load_xor_i64_acq_rel:
4849   %old = atomicrmw xor i64* @var64, i64 %offset acq_rel
4850; CHECK-NOT: dmb
4851; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4852; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4853
4854; CHECK: ldeoral x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4855; CHECK-NOT: dmb
4856
4857   ret i64 %old
4858}
4859
4860define void @test_atomic_load_xor_i32_noret_acq_rel(i32 %offset) nounwind {
4861; CHECK-LABEL: test_atomic_load_xor_i32_noret_acq_rel:
4862   atomicrmw xor i32* @var32, i32 %offset acq_rel
4863; CHECK-NOT: dmb
4864; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4865; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4866
4867; CHECK: ldeoral w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
4868; CHECK-NOT: dmb
4869  ret void
4870}
4871
4872define void @test_atomic_load_xor_i64_noret_acq_rel(i64 %offset) nounwind {
4873; CHECK-LABEL: test_atomic_load_xor_i64_noret_acq_rel:
4874   atomicrmw xor i64* @var64, i64 %offset acq_rel
4875; CHECK-NOT: dmb
4876; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4877; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4878
4879; CHECK: ldeoral x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
4880; CHECK-NOT: dmb
4881  ret void
4882}
4883
4884define i8 @test_atomic_load_xor_i8_acquire(i8 %offset) nounwind {
4885; CHECK-LABEL: test_atomic_load_xor_i8_acquire:
4886   %old = atomicrmw xor i8* @var8, i8 %offset acquire
4887; CHECK-NOT: dmb
4888; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
4889; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
4890
4891; CHECK: ldeorab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4892; CHECK-NOT: dmb
4893
4894   ret i8 %old
4895}
4896
4897define i16 @test_atomic_load_xor_i16_acquire(i16 %offset) nounwind {
4898; CHECK-LABEL: test_atomic_load_xor_i16_acquire:
4899   %old = atomicrmw xor i16* @var16, i16 %offset acquire
4900; CHECK-NOT: dmb
4901; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
4902; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
4903
4904; CHECK: ldeorah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4905; CHECK-NOT: dmb
4906
4907   ret i16 %old
4908}
4909
4910define i32 @test_atomic_load_xor_i32_acquire(i32 %offset) nounwind {
4911; CHECK-LABEL: test_atomic_load_xor_i32_acquire:
4912   %old = atomicrmw xor i32* @var32, i32 %offset acquire
4913; CHECK-NOT: dmb
4914; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4915; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4916
4917; CHECK: ldeora w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4918; CHECK-NOT: dmb
4919
4920   ret i32 %old
4921}
4922
4923define i64 @test_atomic_load_xor_i64_acquire(i64 %offset) nounwind {
4924; CHECK-LABEL: test_atomic_load_xor_i64_acquire:
4925   %old = atomicrmw xor i64* @var64, i64 %offset acquire
4926; CHECK-NOT: dmb
4927; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4928; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4929
4930; CHECK: ldeora x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
4931; CHECK-NOT: dmb
4932
4933   ret i64 %old
4934}
4935
4936define void @test_atomic_load_xor_i32_noret_acquire(i32 %offset) nounwind {
4937; CHECK-LABEL: test_atomic_load_xor_i32_noret_acquire:
4938   atomicrmw xor i32* @var32, i32 %offset acquire
4939; CHECK-NOT: dmb
4940; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4941; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4942
4943; CHECK: ldeora w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
4944; CHECK-NOT: dmb
4945  ret void
4946}
4947
4948define void @test_atomic_load_xor_i64_noret_acquire(i64 %offset) nounwind {
4949; CHECK-LABEL: test_atomic_load_xor_i64_noret_acquire:
4950   atomicrmw xor i64* @var64, i64 %offset acquire
4951; CHECK-NOT: dmb
4952; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
4953; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
4954
4955; CHECK: ldeora x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
4956; CHECK-NOT: dmb
4957  ret void
4958}
4959
4960define i8 @test_atomic_load_xor_i8_monotonic(i8 %offset) nounwind {
4961; CHECK-LABEL: test_atomic_load_xor_i8_monotonic:
4962   %old = atomicrmw xor i8* @var8, i8 %offset monotonic
4963; CHECK-NOT: dmb
4964; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
4965; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
4966
4967; CHECK: ldeorb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4968; CHECK-NOT: dmb
4969
4970   ret i8 %old
4971}
4972
4973define i16 @test_atomic_load_xor_i16_monotonic(i16 %offset) nounwind {
4974; CHECK-LABEL: test_atomic_load_xor_i16_monotonic:
4975   %old = atomicrmw xor i16* @var16, i16 %offset monotonic
4976; CHECK-NOT: dmb
4977; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
4978; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
4979
4980; CHECK: ldeorh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4981; CHECK-NOT: dmb
4982
4983   ret i16 %old
4984}
4985
4986define i32 @test_atomic_load_xor_i32_monotonic(i32 %offset) nounwind {
4987; CHECK-LABEL: test_atomic_load_xor_i32_monotonic:
4988   %old = atomicrmw xor i32* @var32, i32 %offset monotonic
4989; CHECK-NOT: dmb
4990; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
4991; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
4992
4993; CHECK: ldeor w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
4994; CHECK-NOT: dmb
4995
4996   ret i32 %old
4997}
4998
4999define i64 @test_atomic_load_xor_i64_monotonic(i64 %offset) nounwind {
5000; CHECK-LABEL: test_atomic_load_xor_i64_monotonic:
5001   %old = atomicrmw xor i64* @var64, i64 %offset monotonic
5002; CHECK-NOT: dmb
5003; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
5004; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
5005
5006; CHECK: ldeor x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
5007; CHECK-NOT: dmb
5008
5009   ret i64 %old
5010}
5011
5012define void @test_atomic_load_xor_i32_noret_monotonic(i32 %offset) nounwind {
5013; CHECK-LABEL: test_atomic_load_xor_i32_noret_monotonic:
5014   atomicrmw xor i32* @var32, i32 %offset monotonic
5015; CHECK-NOT: dmb
5016; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
5017; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
5018
5019; CHECK: steor w0, [x[[ADDR]]]
5020; CHECK-NOT: dmb
5021  ret void
5022}
5023
5024define void @test_atomic_load_xor_i64_noret_monotonic(i64 %offset) nounwind {
5025; CHECK-LABEL: test_atomic_load_xor_i64_noret_monotonic:
5026   atomicrmw xor i64* @var64, i64 %offset monotonic
5027; CHECK-NOT: dmb
5028; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
5029; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
5030
5031; CHECK: steor x0, [x[[ADDR]]]
5032; CHECK-NOT: dmb
5033  ret void
5034}
5035
5036define i8 @test_atomic_load_xor_i8_release(i8 %offset) nounwind {
5037; CHECK-LABEL: test_atomic_load_xor_i8_release:
5038   %old = atomicrmw xor i8* @var8, i8 %offset release
5039; CHECK-NOT: dmb
5040; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
5041; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
5042
5043; CHECK: ldeorlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
5044; CHECK-NOT: dmb
5045
5046   ret i8 %old
5047}
5048
5049define i16 @test_atomic_load_xor_i16_release(i16 %offset) nounwind {
5050; CHECK-LABEL: test_atomic_load_xor_i16_release:
5051   %old = atomicrmw xor i16* @var16, i16 %offset release
5052; CHECK-NOT: dmb
5053; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
5054; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
5055
5056; CHECK: ldeorlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
5057; CHECK-NOT: dmb
5058
5059   ret i16 %old
5060}
5061
5062define i32 @test_atomic_load_xor_i32_release(i32 %offset) nounwind {
5063; CHECK-LABEL: test_atomic_load_xor_i32_release:
5064   %old = atomicrmw xor i32* @var32, i32 %offset release
5065; CHECK-NOT: dmb
5066; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
5067; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
5068
5069; CHECK: ldeorl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
5070; CHECK-NOT: dmb
5071
5072   ret i32 %old
5073}
5074
5075define i64 @test_atomic_load_xor_i64_release(i64 %offset) nounwind {
5076; CHECK-LABEL: test_atomic_load_xor_i64_release:
5077   %old = atomicrmw xor i64* @var64, i64 %offset release
5078; CHECK-NOT: dmb
5079; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
5080; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
5081
5082; CHECK: ldeorl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
5083; CHECK-NOT: dmb
5084
5085   ret i64 %old
5086}
5087
5088define void @test_atomic_load_xor_i32_noret_release(i32 %offset) nounwind {
5089; CHECK-LABEL: test_atomic_load_xor_i32_noret_release:
5090   atomicrmw xor i32* @var32, i32 %offset release
5091; CHECK-NOT: dmb
5092; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
5093; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
5094
5095; CHECK: steorl w0, [x[[ADDR]]]
5096; CHECK-NOT: dmb
5097  ret void
5098}
5099
5100define void @test_atomic_load_xor_i64_noret_release(i64 %offset) nounwind {
5101; CHECK-LABEL: test_atomic_load_xor_i64_noret_release:
5102   atomicrmw xor i64* @var64, i64 %offset release
5103; CHECK-NOT: dmb
5104; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
5105; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
5106
5107; CHECK: steorl x0, [x[[ADDR]]]
5108; CHECK-NOT: dmb
5109  ret void
5110}
5111
5112define i8 @test_atomic_load_xor_i8_seq_cst(i8 %offset) nounwind {
5113; CHECK-LABEL: test_atomic_load_xor_i8_seq_cst:
5114   %old = atomicrmw xor i8* @var8, i8 %offset seq_cst
5115; CHECK-NOT: dmb
5116; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
5117; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
5118
5119; CHECK: ldeoralb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
5120; CHECK-NOT: dmb
5121
5122   ret i8 %old
5123}
5124
5125define i16 @test_atomic_load_xor_i16_seq_cst(i16 %offset) nounwind {
5126; CHECK-LABEL: test_atomic_load_xor_i16_seq_cst:
5127   %old = atomicrmw xor i16* @var16, i16 %offset seq_cst
5128; CHECK-NOT: dmb
5129; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
5130; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
5131
5132; CHECK: ldeoralh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
5133; CHECK-NOT: dmb
5134
5135   ret i16 %old
5136}
5137
5138define i32 @test_atomic_load_xor_i32_seq_cst(i32 %offset) nounwind {
5139; CHECK-LABEL: test_atomic_load_xor_i32_seq_cst:
5140   %old = atomicrmw xor i32* @var32, i32 %offset seq_cst
5141; CHECK-NOT: dmb
5142; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
5143; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
5144
5145; CHECK: ldeoral w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
5146; CHECK-NOT: dmb
5147
5148   ret i32 %old
5149}
5150
5151define i64 @test_atomic_load_xor_i64_seq_cst(i64 %offset) nounwind {
5152; CHECK-LABEL: test_atomic_load_xor_i64_seq_cst:
5153   %old = atomicrmw xor i64* @var64, i64 %offset seq_cst
5154; CHECK-NOT: dmb
5155; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
5156; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
5157
5158; CHECK: ldeoral x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
5159; CHECK-NOT: dmb
5160
5161   ret i64 %old
5162}
5163
5164define void @test_atomic_load_xor_i32_noret_seq_cst(i32 %offset) nounwind {
5165; CHECK-LABEL: test_atomic_load_xor_i32_noret_seq_cst:
5166   atomicrmw xor i32* @var32, i32 %offset seq_cst
5167; CHECK-NOT: dmb
5168; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
5169; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
5170
5171; CHECK: ldeoral w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
5172; CHECK-NOT: dmb
5173  ret void
5174}
5175
5176define void @test_atomic_load_xor_i64_noret_seq_cst(i64 %offset) nounwind {
5177; CHECK-LABEL: test_atomic_load_xor_i64_noret_seq_cst:
5178   atomicrmw xor i64* @var64, i64 %offset seq_cst
5179; CHECK-NOT: dmb
5180; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
5181; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
5182
5183; CHECK: ldeoral x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
5184; CHECK-NOT: dmb
5185  ret void
5186}
5187
5188
5189