1; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,CIVI %s 2; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,CIVI %s 3; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s 4 5declare i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* nocapture, i32, i32, i32, i1) #2 6declare i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #2 7declare i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* nocapture, i32, i32, i32, i1) #2 8 9declare i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* nocapture, i64, i32, i32, i1) #2 10declare i64 @llvm.amdgcn.atomic.dec.i64.p3i64(i64 addrspace(3)* nocapture, i64, i32, i32, i1) #2 11declare i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* nocapture, i64, i32, i32, i1) #2 12 13declare i32 @llvm.amdgcn.workitem.id.x() #1 14 15; GCN-LABEL: {{^}}lds_atomic_dec_ret_i32: 16; CIVI-DAG: s_mov_b32 m0 17; GFX9-NOT: m0 18 19; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42 20; GCN: ds_dec_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[K]] 21define amdgpu_kernel void @lds_atomic_dec_ret_i32(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) #0 { 22 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 0, i32 0, i1 false) 23 store i32 %result, i32 addrspace(1)* %out 24 ret void 25} 26 27; GCN-LABEL: {{^}}lds_atomic_dec_ret_i32_offset: 28; CIVI-DAG: s_mov_b32 m0 29; GFX9-NOT: m0 30 31; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42 32; GCN: ds_dec_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[K]] offset:16 33define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) #0 { 34 %gep = getelementptr i32, i32 addrspace(3)* %ptr, i32 4 35 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %gep, i32 42, i32 0, i32 0, i1 false) 36 store i32 %result, i32 addrspace(1)* %out 37 ret void 38} 39 40; GCN-LABEL: {{^}}lds_atomic_dec_noret_i32: 41; CIVI-DAG: s_mov_b32 m0 42; GFX9-NOT: m0 43 44; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]], 45; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4 46; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]] 47; GCN: ds_dec_u32 [[VPTR]], [[DATA]] 48define amdgpu_kernel void @lds_atomic_dec_noret_i32(i32 addrspace(3)* %ptr) nounwind { 49 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 0, i32 0, i1 false) 50 ret void 51} 52 53; GCN-LABEL: {{^}}lds_atomic_dec_noret_i32_offset: 54; CIVI-DAG: s_mov_b32 m0 55; GFX9-NOT: m0 56 57; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42 58; GCN: ds_dec_u32 v{{[0-9]+}}, [[K]] offset:16 59define amdgpu_kernel void @lds_atomic_dec_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind { 60 %gep = getelementptr i32, i32 addrspace(3)* %ptr, i32 4 61 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %gep, i32 42, i32 0, i32 0, i1 false) 62 ret void 63} 64 65; GCN-LABEL: {{^}}global_atomic_dec_ret_i32: 66; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42 67; CIVI: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 glc{{$}} 68; GFX9-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} 69; GFX9: global_atomic_dec v{{[0-9]+}}, [[ZERO]], [[K]], s{{\[[0-9]+:[0-9]+\]}} glc{{$}} 70define amdgpu_kernel void @global_atomic_dec_ret_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #0 { 71 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* %ptr, i32 42, i32 0, i32 0, i1 false) 72 store i32 %result, i32 addrspace(1)* %out 73 ret void 74} 75 76; GCN-LABEL: {{^}}global_atomic_dec_ret_i32_offset: 77; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42 78; CIVI: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16 glc{{$}} 79 80; GFX9-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} 81; GFX9: global_atomic_dec v{{[0-9]+}}, [[ZERO]], [[K]], s{{\[[0-9]+:[0-9]+\]}} offset:16 glc{{$}} 82define amdgpu_kernel void @global_atomic_dec_ret_i32_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #0 { 83 %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4 84 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* %gep, i32 42, i32 0, i32 0, i1 false) 85 store i32 %result, i32 addrspace(1)* %out 86 ret void 87} 88 89; GCN-LABEL: {{^}}global_atomic_dec_noret_i32: 90; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42 91; CIVI: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} 92 93; GFX9-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} 94; GFX9: global_atomic_dec [[ZERO]], [[K]], s{{\[[0-9]+:[0-9]+\]$}} 95define amdgpu_kernel void @global_atomic_dec_noret_i32(i32 addrspace(1)* %ptr) nounwind { 96 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* %ptr, i32 42, i32 0, i32 0, i1 false) 97 ret void 98} 99 100; GCN-LABEL: {{^}}global_atomic_dec_noret_i32_offset: 101; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42 102; CIVI: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16{{$}} 103 104; GFX9-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} 105; GFX9: global_atomic_dec [[ZERO]], [[K]], s{{\[[0-9]+:[0-9]+\]}} offset:16{{$}} 106define amdgpu_kernel void @global_atomic_dec_noret_i32_offset(i32 addrspace(1)* %ptr) nounwind { 107 %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4 108 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* %gep, i32 42, i32 0, i32 0, i1 false) 109 ret void 110} 111 112; GCN-LABEL: {{^}}global_atomic_dec_ret_i32_offset_addr64: 113; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 114; CI: buffer_atomic_dec [[K]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:20 glc{{$}} 115; VI: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}} 116define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #0 { 117 %id = call i32 @llvm.amdgcn.workitem.id.x() 118 %gep.tid = getelementptr i32, i32 addrspace(1)* %ptr, i32 %id 119 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %id 120 %gep = getelementptr i32, i32 addrspace(1)* %gep.tid, i32 5 121 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* %gep, i32 42, i32 0, i32 0, i1 false) 122 store i32 %result, i32 addrspace(1)* %out.gep 123 ret void 124} 125 126; GCN-LABEL: {{^}}global_atomic_dec_noret_i32_offset_addr64: 127; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 128; CI: buffer_atomic_dec [[K]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:20{{$}} 129; VI: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}} 130define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_addr64(i32 addrspace(1)* %ptr) #0 { 131 %id = call i32 @llvm.amdgcn.workitem.id.x() 132 %gep.tid = getelementptr i32, i32 addrspace(1)* %ptr, i32 %id 133 %gep = getelementptr i32, i32 addrspace(1)* %gep.tid, i32 5 134 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* %gep, i32 42, i32 0, i32 0, i1 false) 135 ret void 136} 137 138; GCN-LABEL: {{^}}flat_atomic_dec_ret_i32: 139; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 140; GCN: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}} 141define amdgpu_kernel void @flat_atomic_dec_ret_i32(i32* %out, i32* %ptr) #0 { 142 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %ptr, i32 42, i32 0, i32 0, i1 false) 143 store i32 %result, i32* %out 144 ret void 145} 146 147; GCN-LABEL: {{^}}flat_atomic_dec_ret_i32_offset: 148; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 149; CIVI: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}} 150; GFX9: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:16 glc{{$}} 151define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(i32* %out, i32* %ptr) #0 { 152 %gep = getelementptr i32, i32* %ptr, i32 4 153 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %gep, i32 42, i32 0, i32 0, i1 false) 154 store i32 %result, i32* %out 155 ret void 156} 157 158; GCN-LABEL: {{^}}flat_atomic_dec_noret_i32: 159; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 160; GCN: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}} 161define amdgpu_kernel void @flat_atomic_dec_noret_i32(i32* %ptr) nounwind { 162 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %ptr, i32 42, i32 0, i32 0, i1 false) 163 ret void 164} 165 166; GCN-LABEL: {{^}}flat_atomic_dec_noret_i32_offset: 167; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 168; CIVI: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}} 169; GFX9: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:16{{$}} 170define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(i32* %ptr) nounwind { 171 %gep = getelementptr i32, i32* %ptr, i32 4 172 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %gep, i32 42, i32 0, i32 0, i1 false) 173 ret void 174} 175 176; GCN-LABEL: {{^}}flat_atomic_dec_ret_i32_offset_addr64: 177; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 178; CIVI: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}} 179; GFX9: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:20 glc{{$}} 180define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(i32* %out, i32* %ptr) #0 { 181 %id = call i32 @llvm.amdgcn.workitem.id.x() 182 %gep.tid = getelementptr i32, i32* %ptr, i32 %id 183 %out.gep = getelementptr i32, i32* %out, i32 %id 184 %gep = getelementptr i32, i32* %gep.tid, i32 5 185 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %gep, i32 42, i32 0, i32 0, i1 false) 186 store i32 %result, i32* %out.gep 187 ret void 188} 189 190; GCN-LABEL: {{^}}flat_atomic_dec_noret_i32_offset_addr64: 191; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 192; CIVI: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}} 193; GFX9: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:20{{$}} 194define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(i32* %ptr) #0 { 195 %id = call i32 @llvm.amdgcn.workitem.id.x() 196 %gep.tid = getelementptr i32, i32* %ptr, i32 %id 197 %gep = getelementptr i32, i32* %gep.tid, i32 5 198 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %gep, i32 42, i32 0, i32 0, i1 false) 199 ret void 200} 201 202; GCN-LABEL: {{^}}flat_atomic_dec_ret_i64: 203; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 204; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 205; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}} 206define amdgpu_kernel void @flat_atomic_dec_ret_i64(i64* %out, i64* %ptr) #0 { 207 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %ptr, i64 42, i32 0, i32 0, i1 false) 208 store i64 %result, i64* %out 209 ret void 210} 211 212; GCN-LABEL: {{^}}flat_atomic_dec_ret_i64_offset: 213; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 214; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 215; CIVI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}} 216; GFX9: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:32 glc{{$}} 217define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(i64* %out, i64* %ptr) #0 { 218 %gep = getelementptr i64, i64* %ptr, i32 4 219 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %gep, i64 42, i32 0, i32 0, i1 false) 220 store i64 %result, i64* %out 221 ret void 222} 223 224; GCN-LABEL: {{^}}flat_atomic_dec_noret_i64: 225; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 226; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 227; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}} 228define amdgpu_kernel void @flat_atomic_dec_noret_i64(i64* %ptr) nounwind { 229 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %ptr, i64 42, i32 0, i32 0, i1 false) 230 ret void 231} 232 233; GCN-LABEL: {{^}}flat_atomic_dec_noret_i64_offset: 234; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 235; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 236; CIVI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}} 237; GFX9: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:32{{$}} 238define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(i64* %ptr) nounwind { 239 %gep = getelementptr i64, i64* %ptr, i32 4 240 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %gep, i64 42, i32 0, i32 0, i1 false) 241 ret void 242} 243 244; GCN-LABEL: {{^}}flat_atomic_dec_ret_i64_offset_addr64: 245; GCN: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 246; GCN: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 247; CIVI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}} 248; GFX9: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:40 glc{{$}} 249define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(i64* %out, i64* %ptr) #0 { 250 %id = call i32 @llvm.amdgcn.workitem.id.x() 251 %gep.tid = getelementptr i64, i64* %ptr, i32 %id 252 %out.gep = getelementptr i64, i64* %out, i32 %id 253 %gep = getelementptr i64, i64* %gep.tid, i32 5 254 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %gep, i64 42, i32 0, i32 0, i1 false) 255 store i64 %result, i64* %out.gep 256 ret void 257} 258 259; GCN-LABEL: {{^}}flat_atomic_dec_noret_i64_offset_addr64: 260; GCN: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 261; GCN: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 262; CIVI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}} 263; GFX9: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:40{{$}} 264define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(i64* %ptr) #0 { 265 %id = call i32 @llvm.amdgcn.workitem.id.x() 266 %gep.tid = getelementptr i64, i64* %ptr, i32 %id 267 %gep = getelementptr i64, i64* %gep.tid, i32 5 268 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %gep, i64 42, i32 0, i32 0, i1 false) 269 ret void 270} 271 272@lds0 = addrspace(3) global [512 x i32] undef 273 274; GCN-LABEL: {{^}}atomic_dec_shl_base_lds_0: 275; CIVI-DAG: s_mov_b32 m0 276; GFX9-NOT: m0 277 278; CIVI-DAG: v_lshlrev_b32_e32 [[OFS:v[0-9]+]], 2, {{v[0-9]+}} 279; CIVI-DAG: v_add_{{[ui]}}32_e32 [[PTR:v[0-9]+]], vcc, lds0@abs32@lo, [[OFS]] 280; GFX9-DAG: s_mov_b32 [[BASE:s[0-9]+]], lds0@abs32@lo 281; GFX9-DAG: v_lshl_add_u32 [[PTR:v[0-9]+]], {{v[0-9]+}}, 2, [[BASE]] 282 283; GCN: ds_dec_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8 284define amdgpu_kernel void @atomic_dec_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 285 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 286 %idx.0 = add nsw i32 %tid.x, 2 287 %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds0, i32 0, i32 %idx.0 288 %val0 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %arrayidx0, i32 9, i32 0, i32 0, i1 false) 289 store i32 %idx.0, i32 addrspace(1)* %add_use 290 store i32 %val0, i32 addrspace(1)* %out 291 ret void 292} 293 294; GCN-LABEL: {{^}}lds_atomic_dec_ret_i64: 295; CIVI-DAG: s_mov_b32 m0 296; GFX9-NOT: m0 297 298; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 299; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 300; GCN: ds_dec_rtn_u64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}{{$}} 301define amdgpu_kernel void @lds_atomic_dec_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) #0 { 302 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p3i64(i64 addrspace(3)* %ptr, i64 42, i32 0, i32 0, i1 false) 303 store i64 %result, i64 addrspace(1)* %out 304 ret void 305} 306 307; GCN-LABEL: {{^}}lds_atomic_dec_ret_i64_offset: 308; CIVI-DAG: s_mov_b32 m0 309; GFX9-NOT: m0 310 311; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 312; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 313; GCN: ds_dec_rtn_u64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:32 314define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) #0 { 315 %gep = getelementptr i64, i64 addrspace(3)* %ptr, i32 4 316 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p3i64(i64 addrspace(3)* %gep, i64 42, i32 0, i32 0, i1 false) 317 store i64 %result, i64 addrspace(1)* %out 318 ret void 319} 320 321; GCN-LABEL: {{^}}lds_atomic_dec_noret_i64: 322; CIVI-DAG: s_mov_b32 m0 323; GFX9-NOT: m0 324 325; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 326; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 327; GCN: ds_dec_u64 v{{[0-9]+}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}{{$}} 328define amdgpu_kernel void @lds_atomic_dec_noret_i64(i64 addrspace(3)* %ptr) nounwind { 329 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p3i64(i64 addrspace(3)* %ptr, i64 42, i32 0, i32 0, i1 false) 330 ret void 331} 332 333; GCN-LABEL: {{^}}lds_atomic_dec_noret_i64_offset: 334; CIVI-DAG: s_mov_b32 m0 335; GFX9-NOT: m0 336 337; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 338; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 339; GCN: ds_dec_u64 v{{[0-9]+}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:32{{$}} 340define amdgpu_kernel void @lds_atomic_dec_noret_i64_offset(i64 addrspace(3)* %ptr) nounwind { 341 %gep = getelementptr i64, i64 addrspace(3)* %ptr, i32 4 342 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p3i64(i64 addrspace(3)* %gep, i64 42, i32 0, i32 0, i1 false) 343 ret void 344} 345 346; GCN-LABEL: {{^}}global_atomic_dec_ret_i64: 347; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 348; GFX9-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}} 349; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 350; CIVI: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 glc{{$}} 351 352; GFX9: global_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v[[ZERO]], v{{\[}}[[KLO]]:[[KHI]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}} glc{{$}} 353define amdgpu_kernel void @global_atomic_dec_ret_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %ptr) #0 { 354 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* %ptr, i64 42, i32 0, i32 0, i1 false) 355 store i64 %result, i64 addrspace(1)* %out 356 ret void 357} 358 359; GCN-LABEL: {{^}}global_atomic_dec_ret_i64_offset: 360; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 361; GFX9-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}} 362; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 363; CIVI: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:32 glc{{$}} 364; GFX9: global_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v[[ZERO]], v{{\[}}[[KLO]]:[[KHI]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}} offset:32 glc{{$}} 365define amdgpu_kernel void @global_atomic_dec_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %ptr) #0 { 366 %gep = getelementptr i64, i64 addrspace(1)* %ptr, i32 4 367 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* %gep, i64 42, i32 0, i32 0, i1 false) 368 store i64 %result, i64 addrspace(1)* %out 369 ret void 370} 371 372; GCN-LABEL: {{^}}global_atomic_dec_noret_i64: 373; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 374; GFX9-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}} 375; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 376; CIVI: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} 377; GFX9: global_atomic_dec_x2 v[[ZERO]], v{{\[}}[[KLO]]:[[KHI]]{{\]}}, s{{\[[0-9]+:[0-9]+\]$}} 378define amdgpu_kernel void @global_atomic_dec_noret_i64(i64 addrspace(1)* %ptr) nounwind { 379 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* %ptr, i64 42, i32 0, i32 0, i1 false) 380 ret void 381} 382 383; GCN-LABEL: {{^}}global_atomic_dec_noret_i64_offset: 384; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 385; GFX9-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}} 386; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 387; CIVI: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:32{{$}} 388; GFX9: global_atomic_dec_x2 v[[ZERO]], v{{\[}}[[KLO]]:[[KHI]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}} offset:32{{$}} 389define amdgpu_kernel void @global_atomic_dec_noret_i64_offset(i64 addrspace(1)* %ptr) nounwind { 390 %gep = getelementptr i64, i64 addrspace(1)* %ptr, i32 4 391 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* %gep, i64 42, i32 0, i32 0, i1 false) 392 ret void 393} 394 395; GCN-LABEL: {{^}}global_atomic_dec_ret_i64_offset_addr64: 396; GCN: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 397; CI: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}} 398; GCN: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 399; CI: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:40 glc{{$}} 400; VI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}} 401define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %ptr) #0 { 402 %id = call i32 @llvm.amdgcn.workitem.id.x() 403 %gep.tid = getelementptr i64, i64 addrspace(1)* %ptr, i32 %id 404 %out.gep = getelementptr i64, i64 addrspace(1)* %out, i32 %id 405 %gep = getelementptr i64, i64 addrspace(1)* %gep.tid, i32 5 406 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* %gep, i64 42, i32 0, i32 0, i1 false) 407 store i64 %result, i64 addrspace(1)* %out.gep 408 ret void 409} 410 411; GCN-LABEL: {{^}}global_atomic_dec_noret_i64_offset_addr64: 412; GCN: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 413; CI: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}} 414; GCN: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} 415; CI: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:40{{$}} 416; VI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}{{$}} 417define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(i64 addrspace(1)* %ptr) #0 { 418 %id = call i32 @llvm.amdgcn.workitem.id.x() 419 %gep.tid = getelementptr i64, i64 addrspace(1)* %ptr, i32 %id 420 %gep = getelementptr i64, i64 addrspace(1)* %gep.tid, i32 5 421 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* %gep, i64 42, i32 0, i32 0, i1 false) 422 ret void 423} 424 425@lds1 = addrspace(3) global [512 x i64] undef, align 8 426 427; GCN-LABEL: {{^}}atomic_dec_shl_base_lds_0_i64: 428; CIVI-DAG: s_mov_b32 m0 429; GFX9-NOT: m0 430 431; CIVI-DAG: v_lshlrev_b32_e32 [[OFS:v[0-9]+]], 3, {{v[0-9]+}} 432; CIVI-DAG: v_add_{{[ui]}}32_e32 [[PTR:v[0-9]+]], vcc, lds1@abs32@lo, [[OFS]] 433; GFX9-DAG: v_mov_b32_e32 [[BASE:v[0-9]+]], lds1@abs32@lo 434; GFX9-DAG: v_lshl_add_u32 [[PTR:v[0-9]+]], {{v[0-9]+}}, 3, [[BASE]] 435 436; GCN: ds_dec_rtn_u64 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]], v{{\[[0-9]+:[0-9]+\]}} offset:16 437define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 438 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 439 %idx.0 = add nsw i32 %tid.x, 2 440 %arrayidx0 = getelementptr inbounds [512 x i64], [512 x i64] addrspace(3)* @lds1, i32 0, i32 %idx.0 441 %val0 = call i64 @llvm.amdgcn.atomic.dec.i64.p3i64(i64 addrspace(3)* %arrayidx0, i64 9, i32 0, i32 0, i1 false) 442 store i32 %idx.0, i32 addrspace(1)* %add_use 443 store i64 %val0, i64 addrspace(1)* %out 444 ret void 445} 446 447attributes #0 = { nounwind } 448attributes #1 = { nounwind readnone } 449attributes #2 = { nounwind argmemonly } 450