1; RUN: llc < %s -march=arm64 -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone | FileCheck %s 2 3@var = global i128 0 4 5define i128 @val_compare_and_swap(i128* %p, i128 %oldval, i128 %newval) { 6; CHECK-LABEL: val_compare_and_swap: 7; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 8; CHECK: ldaxp [[RESULTLO:x[0-9]+]], [[RESULTHI:x[0-9]+]], [x[[ADDR:[0-9]+]]] 9; CHECK-DAG: eor [[MISMATCH_LO:x[0-9]+]], [[RESULTLO]], x2 10; CHECK-DAG: eor [[MISMATCH_HI:x[0-9]+]], [[RESULTHI]], x3 11; CHECK: orr [[MISMATCH:x[0-9]+]], [[MISMATCH_LO]], [[MISMATCH_HI]] 12; CHECK: cbnz [[MISMATCH]], [[DONE:.LBB[0-9]+_[0-9]+]] 13; CHECK: stxp [[SCRATCH_RES:w[0-9]+]], x4, x5, [x[[ADDR]]] 14; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 15; CHECK: [[DONE]]: 16 %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire 17 %val = extractvalue { i128, i1 } %pair, 0 18 ret i128 %val 19} 20 21define void @fetch_and_nand(i128* %p, i128 %bits) { 22; CHECK-LABEL: fetch_and_nand: 23; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 24; CHECK: ldxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 25; CHECK-DAG: and [[TMP_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 26; CHECK-DAG: and [[TMP_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 27; CHECK-DAG: mvn [[SCRATCH_REGLO:x[0-9]+]], [[TMP_REGLO]] 28; CHECK-DAG: mvn [[SCRATCH_REGHI:x[0-9]+]], [[TMP_REGHI]] 29; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 30; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 31 32; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 33 %val = atomicrmw nand i128* %p, i128 %bits release 34 store i128 %val, i128* @var, align 16 35 ret void 36} 37 38define void @fetch_and_or(i128* %p, i128 %bits) { 39; CHECK-LABEL: fetch_and_or: 40; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 41; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 42; CHECK-DAG: orr [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 43; CHECK-DAG: orr [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 44; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 45; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 46 47; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 48 %val = atomicrmw or i128* %p, i128 %bits seq_cst 49 store i128 %val, i128* @var, align 16 50 ret void 51} 52 53define void @fetch_and_add(i128* %p, i128 %bits) { 54; CHECK-LABEL: fetch_and_add: 55; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 56; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 57; CHECK: adds [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 58; CHECK: adcs [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 59; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 60; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 61 62; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 63 %val = atomicrmw add i128* %p, i128 %bits seq_cst 64 store i128 %val, i128* @var, align 16 65 ret void 66} 67 68define void @fetch_and_sub(i128* %p, i128 %bits) { 69; CHECK-LABEL: fetch_and_sub: 70; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 71; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 72; CHECK: subs [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 73; CHECK: sbcs [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 74; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 75; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 76 77; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 78 %val = atomicrmw sub i128* %p, i128 %bits seq_cst 79 store i128 %val, i128* @var, align 16 80 ret void 81} 82 83define void @fetch_and_min(i128* %p, i128 %bits) { 84; CHECK-LABEL: fetch_and_min: 85; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 86; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 87; CHECK: cmp [[DEST_REGLO]], x2 88; CHECK: cset [[LOCMP:w[0-9]+]], ls 89; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 90; CHECK: cset [[HICMP:w[0-9]+]], le 91; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq 92; CHECK: cmp [[CMP]], #0 93; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne 94; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne 95; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 96; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 97 98; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 99 %val = atomicrmw min i128* %p, i128 %bits seq_cst 100 store i128 %val, i128* @var, align 16 101 ret void 102} 103 104define void @fetch_and_max(i128* %p, i128 %bits) { 105; CHECK-LABEL: fetch_and_max: 106; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 107; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 108; CHECK: cmp [[DEST_REGLO]], x2 109; CHECK: cset [[LOCMP:w[0-9]+]], hi 110; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 111; CHECK: cset [[HICMP:w[0-9]+]], gt 112; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq 113; CHECK: cmp [[CMP]], #0 114; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne 115; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne 116; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 117; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 118 119; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 120 %val = atomicrmw max i128* %p, i128 %bits seq_cst 121 store i128 %val, i128* @var, align 16 122 ret void 123} 124 125define void @fetch_and_umin(i128* %p, i128 %bits) { 126; CHECK-LABEL: fetch_and_umin: 127; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 128; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 129; CHECK: cmp [[DEST_REGLO]], x2 130; CHECK: cset [[LOCMP:w[0-9]+]], ls 131; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 132; CHECK: cset [[HICMP:w[0-9]+]], ls 133; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq 134; CHECK: cmp [[CMP]], #0 135; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne 136; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne 137; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 138; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 139 140; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 141 %val = atomicrmw umin i128* %p, i128 %bits seq_cst 142 store i128 %val, i128* @var, align 16 143 ret void 144} 145 146define void @fetch_and_umax(i128* %p, i128 %bits) { 147; CHECK-LABEL: fetch_and_umax: 148; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 149; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 150; CHECK: cmp [[DEST_REGLO]], x2 151; CHECK: cset [[LOCMP:w[0-9]+]], hi 152; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 153; CHECK: cset [[HICMP:w[0-9]+]], hi 154; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq 155; CHECK: cmp [[CMP]], #0 156; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne 157; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne 158; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 159; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 160 161; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 162 %val = atomicrmw umax i128* %p, i128 %bits seq_cst 163 store i128 %val, i128* @var, align 16 164 ret void 165} 166 167define i128 @atomic_load_seq_cst(i128* %p) { 168; CHECK-LABEL: atomic_load_seq_cst: 169; CHECK-NOT: dmb 170; CHECK-LABEL: ldaxp 171; CHECK-NOT: dmb 172 %r = load atomic i128, i128* %p seq_cst, align 16 173 ret i128 %r 174} 175 176define i128 @atomic_load_relaxed(i128* %p) { 177; CHECK-LABEL: atomic_load_relaxed: 178; CHECK-NOT: dmb 179; CHECK: ldxp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0] 180; CHECK-NOT: dmb 181 %r = load atomic i128, i128* %p monotonic, align 16 182 ret i128 %r 183} 184 185 186define void @atomic_store_seq_cst(i128 %in, i128* %p) { 187; CHECK-LABEL: atomic_store_seq_cst: 188; CHECK-NOT: dmb 189; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 190; CHECK: ldaxp xzr, xzr, [x2] 191; CHECK: stlxp [[SUCCESS:w[0-9]+]], x0, x1, [x2] 192; CHECK: cbnz [[SUCCESS]], [[LABEL]] 193; CHECK-NOT: dmb 194 store atomic i128 %in, i128* %p seq_cst, align 16 195 ret void 196} 197 198define void @atomic_store_release(i128 %in, i128* %p) { 199; CHECK-LABEL: atomic_store_release: 200; CHECK-NOT: dmb 201; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 202; CHECK: ldxp xzr, xzr, [x2] 203; CHECK: stlxp [[SUCCESS:w[0-9]+]], x0, x1, [x2] 204; CHECK: cbnz [[SUCCESS]], [[LABEL]] 205; CHECK-NOT: dmb 206 store atomic i128 %in, i128* %p release, align 16 207 ret void 208} 209 210define void @atomic_store_relaxed(i128 %in, i128* %p) { 211; CHECK-LABEL: atomic_store_relaxed: 212; CHECK-NOT: dmb 213; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 214; CHECK: ldxp xzr, xzr, [x2] 215; CHECK: stxp [[SUCCESS:w[0-9]+]], x0, x1, [x2] 216; CHECK: cbnz [[SUCCESS]], [[LABEL]] 217; CHECK-NOT: dmb 218 store atomic i128 %in, i128* %p unordered, align 16 219 ret void 220} 221