1; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s 2; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s 3; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s 4; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX10 %s 5; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32,GFX10 %s 6 7declare i32 @llvm.amdgcn.workitem.id.x() 8declare i32 @llvm.amdgcn.struct.buffer.atomic.add(i32, <4 x i32>, i32, i32, i32, i32) 9declare i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32, <4 x i32>, i32, i32, i32, i32) 10 11; Show what the atomic optimization pass will do for struct buffers. 12 13; GCN-LABEL: add_i32_constant: 14; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 15; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 16; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 17; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 18; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 19; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 20; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 21; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 22; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5 23; GCN: buffer_atomic_add v[[value]] 24define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out, <4 x i32> %inout) { 25entry: 26 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.add(i32 5, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 0) 27 store i32 %old, i32 addrspace(1)* %out 28 ret void 29} 30 31; GCN-LABEL: add_i32_uniform: 32; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 33; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 34; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 35; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 36; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 37; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 38; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 39; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 40; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]] 41; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]] 42; GCN: buffer_atomic_add v[[value]] 43define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, <4 x i32> %inout, i32 %additive) { 44entry: 45 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.add(i32 %additive, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 0) 46 store i32 %old, i32 addrspace(1)* %out 47 ret void 48} 49 50; GCN-LABEL: add_i32_varying_vdata: 51; GFX7LESS-NOT: v_mbcnt_lo_u32_b32 52; GFX7LESS-NOT: v_mbcnt_hi_u32_b32 53; GFX7LESS-NOT: s_bcnt1_i32_b64 54; GFX7LESS: buffer_atomic_add v{{[0-9]+}} 55; DPPCOMB: v_add_u32_dpp 56; DPPCOMB: v_add_u32_dpp 57; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31 58; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63 59; GFX89: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]] 60; GFX10: s_mov_b32 s[[copy_value:[0-9]+]], s[[scalar_value]] 61; GFX10: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[copy_value]] 62; GFX8MORE: buffer_atomic_add v[[value]] 63define amdgpu_kernel void @add_i32_varying_vdata(i32 addrspace(1)* %out, <4 x i32> %inout) { 64entry: 65 %lane = call i32 @llvm.amdgcn.workitem.id.x() 66 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.add(i32 %lane, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 0) 67 store i32 %old, i32 addrspace(1)* %out 68 ret void 69} 70 71; GCN-LABEL: add_i32_varying_vindex: 72; GCN-NOT: v_mbcnt_lo_u32_b32 73; GCN-NOT: v_mbcnt_hi_u32_b32 74; GCN-NOT: s_bcnt1_i32_b64 75; GCN: buffer_atomic_add v{{[0-9]+}} 76define amdgpu_kernel void @add_i32_varying_vindex(i32 addrspace(1)* %out, <4 x i32> %inout) { 77entry: 78 %lane = call i32 @llvm.amdgcn.workitem.id.x() 79 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.add(i32 1, <4 x i32> %inout, i32 %lane, i32 0, i32 0, i32 0) 80 store i32 %old, i32 addrspace(1)* %out 81 ret void 82} 83 84; GCN-LABEL: add_i32_varying_offset: 85; GCN-NOT: v_mbcnt_lo_u32_b32 86; GCN-NOT: v_mbcnt_hi_u32_b32 87; GCN-NOT: s_bcnt1_i32_b64 88; GCN: buffer_atomic_add v{{[0-9]+}} 89define amdgpu_kernel void @add_i32_varying_offset(i32 addrspace(1)* %out, <4 x i32> %inout) { 90entry: 91 %lane = call i32 @llvm.amdgcn.workitem.id.x() 92 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.add(i32 1, <4 x i32> %inout, i32 0, i32 %lane, i32 0, i32 0) 93 store i32 %old, i32 addrspace(1)* %out 94 ret void 95} 96 97; GCN-LABEL: sub_i32_constant: 98; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 99; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 100; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 101; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 102; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 103; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 104; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 105; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 106; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5 107; GCN: buffer_atomic_sub v[[value]] 108define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out, <4 x i32> %inout) { 109entry: 110 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32 5, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 0) 111 store i32 %old, i32 addrspace(1)* %out 112 ret void 113} 114 115; GCN-LABEL: sub_i32_uniform: 116; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 117; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 118; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 119; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 120; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 121; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 122; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 123; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 124; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]] 125; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]] 126; GCN: buffer_atomic_sub v[[value]] 127define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, <4 x i32> %inout, i32 %subitive) { 128entry: 129 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32 %subitive, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 0) 130 store i32 %old, i32 addrspace(1)* %out 131 ret void 132} 133 134; GCN-LABEL: sub_i32_varying_vdata: 135; GFX7LESS-NOT: v_mbcnt_lo_u32_b32 136; GFX7LESS-NOT: v_mbcnt_hi_u32_b32 137; GFX7LESS-NOT: s_bcnt1_i32_b64 138; GFX7LESS: buffer_atomic_sub v{{[0-9]+}} 139; DPPCOMB: v_add_u32_dpp 140; DPPCOMB: v_add_u32_dpp 141; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31 142; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63 143; GFX89: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]] 144; GFX10: s_mov_b32 s[[copy_value:[0-9]+]], s[[scalar_value]] 145; GFX10: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[copy_value]] 146; GFX8MORE: buffer_atomic_sub v[[value]] 147define amdgpu_kernel void @sub_i32_varying_vdata(i32 addrspace(1)* %out, <4 x i32> %inout) { 148entry: 149 %lane = call i32 @llvm.amdgcn.workitem.id.x() 150 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32 %lane, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 0) 151 store i32 %old, i32 addrspace(1)* %out 152 ret void 153} 154 155; GCN-LABEL: sub_i32_varying_vindex: 156; GCN-NOT: v_mbcnt_lo_u32_b32 157; GCN-NOT: v_mbcnt_hi_u32_b32 158; GCN-NOT: s_bcnt1_i32_b64 159; GCN: buffer_atomic_sub v{{[0-9]+}} 160define amdgpu_kernel void @sub_i32_varying_vindex(i32 addrspace(1)* %out, <4 x i32> %inout) { 161entry: 162 %lane = call i32 @llvm.amdgcn.workitem.id.x() 163 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32 1, <4 x i32> %inout, i32 %lane, i32 0, i32 0, i32 0) 164 store i32 %old, i32 addrspace(1)* %out 165 ret void 166} 167 168; GCN-LABEL: sub_i32_varying_offset: 169; GCN-NOT: v_mbcnt_lo_u32_b32 170; GCN-NOT: v_mbcnt_hi_u32_b32 171; GCN-NOT: s_bcnt1_i32_b64 172; GCN: buffer_atomic_sub v{{[0-9]+}} 173define amdgpu_kernel void @sub_i32_varying_offset(i32 addrspace(1)* %out, <4 x i32> %inout) { 174entry: 175 %lane = call i32 @llvm.amdgcn.workitem.id.x() 176 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32 1, <4 x i32> %inout, i32 0, i32 %lane, i32 0, i32 0) 177 store i32 %old, i32 addrspace(1)* %out 178 ret void 179} 180