1; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -check-prefix=CI %s
2
3@lds = addrspace(3) global [512 x float] undef, align 4
4@lds.v2 = addrspace(3) global [512 x <2 x float>] undef, align 4
5@lds.v3 = addrspace(3) global [512 x <3 x float>] undef, align 4
6@lds.v4 = addrspace(3) global [512 x <4 x float>] undef, align 4
7@lds.v8 = addrspace(3) global [512 x <8 x float>] undef, align 4
8@lds.v16 = addrspace(3) global [512 x <16 x float>] undef, align 4
9
10; CI-LABEL: {{^}}simple_read2_v2f32_superreg_align4:
11; CI: ds_read2_b32 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}} offset1:1{{$}}
12; CI: s_waitcnt lgkmcnt(0)
13; CI: buffer_store_dwordx2 [[RESULT]]
14; CI: s_endpgm
15define void @simple_read2_v2f32_superreg_align4(<2 x float> addrspace(1)* %out) #0 {
16  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
17  %arrayidx0 = getelementptr inbounds  [512 x <2 x float>], [512 x <2 x float>] addrspace(3)* @lds.v2, i32 0, i32 %x.i
18  %val0 = load <2 x float>, <2 x float> addrspace(3)* %arrayidx0, align 4
19  %out.gep = getelementptr inbounds <2 x float>, <2 x float> addrspace(1)* %out, i32 %x.i
20  store <2 x float> %val0, <2 x float> addrspace(1)* %out.gep
21  ret void
22}
23
24; CI-LABEL: {{^}}simple_read2_v2f32_superreg:
25; CI: ds_read_b64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}{{$}}
26; CI: s_waitcnt lgkmcnt(0)
27; CI: buffer_store_dwordx2 [[RESULT]]
28; CI: s_endpgm
29define void @simple_read2_v2f32_superreg(<2 x float> addrspace(1)* %out) #0 {
30  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
31  %arrayidx0 = getelementptr inbounds [512 x <2 x float>], [512 x <2 x float>] addrspace(3)* @lds.v2, i32 0, i32 %x.i
32  %val0 = load <2 x float>, <2 x float> addrspace(3)* %arrayidx0
33  %out.gep = getelementptr inbounds <2 x float>, <2 x float> addrspace(1)* %out, i32 %x.i
34  store <2 x float> %val0, <2 x float> addrspace(1)* %out.gep
35  ret void
36}
37
38; CI-LABEL: {{^}}simple_read2_v4f32_superreg_align4:
39; CI-DAG: ds_read2_b32 v{{\[}}[[REG_X:[0-9]+]]:[[REG_Y:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1{{$}}
40; CI-DAG: ds_read2_b32 v{{\[}}[[REG_Z:[0-9]+]]:[[REG_W:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
41; CI-DAG: v_add_f32_e32 v[[ADD0:[0-9]+]], v[[REG_Z]], v[[REG_X]]
42; CI-DAG: v_add_f32_e32 v[[ADD1:[0-9]+]], v[[REG_W]], v[[REG_Y]]
43; CI: v_add_f32_e32 v[[ADD2:[0-9]+]], v[[ADD1]], v[[ADD0]]
44; CI: buffer_store_dword v[[ADD2]]
45; CI: s_endpgm
46define void @simple_read2_v4f32_superreg_align4(float addrspace(1)* %out) #0 {
47  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
48  %arrayidx0 = getelementptr inbounds [512 x <4 x float>], [512 x <4 x float>] addrspace(3)* @lds.v4, i32 0, i32 %x.i
49  %val0 = load <4 x float>, <4 x float> addrspace(3)* %arrayidx0, align 4
50  %elt0 = extractelement <4 x float> %val0, i32 0
51  %elt1 = extractelement <4 x float> %val0, i32 1
52  %elt2 = extractelement <4 x float> %val0, i32 2
53  %elt3 = extractelement <4 x float> %val0, i32 3
54
55  %add0 = fadd float %elt0, %elt2
56  %add1 = fadd float %elt1, %elt3
57  %add2 = fadd float %add0, %add1
58
59  %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i
60  store float %add2, float addrspace(1)* %out.gep
61  ret void
62}
63
64; CI-LABEL: {{^}}simple_read2_v3f32_superreg_align4:
65; CI-DAG: ds_read2_b32 v{{\[}}[[REG_X:[0-9]+]]:[[REG_Y:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1{{$}}
66; CI-DAG: ds_read_b32 v[[REG_Z:[0-9]+]], v{{[0-9]+}} offset:8{{$}}
67; CI-DAG: v_add_f32_e32 v[[ADD0:[0-9]+]], v[[REG_Z]], v[[REG_X]]
68; CI-DAG: v_add_f32_e32 v[[ADD1:[0-9]+]], v[[REG_Y]], v[[ADD0]]
69; CI: buffer_store_dword v[[ADD1]]
70; CI: s_endpgm
71define void @simple_read2_v3f32_superreg_align4(float addrspace(1)* %out) #0 {
72  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
73  %arrayidx0 = getelementptr inbounds [512 x <3 x float>], [512 x <3 x float>] addrspace(3)* @lds.v3, i32 0, i32 %x.i
74  %val0 = load <3 x float>, <3 x float> addrspace(3)* %arrayidx0, align 4
75  %elt0 = extractelement <3 x float> %val0, i32 0
76  %elt1 = extractelement <3 x float> %val0, i32 1
77  %elt2 = extractelement <3 x float> %val0, i32 2
78
79  %add0 = fadd float %elt0, %elt2
80  %add1 = fadd float %add0, %elt1
81
82  %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i
83  store float %add1, float addrspace(1)* %out.gep
84  ret void
85}
86
87; CI-LABEL: {{^}}simple_read2_v4f32_superreg_align8:
88; CI-DAG: ds_read2_b64 v{{\[}}[[REG_W:[0-9]+]]:[[REG_Z:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1{{$}}
89
90; FIXME: These moves shouldn't be necessary, it should be able to
91; store the same register if offset1 was the non-zero offset.
92
93; CI: v_mov_b32
94; CI: v_mov_b32
95; CI: buffer_store_dwordx4
96; CI: s_endpgm
97define void @simple_read2_v4f32_superreg_align8(<4 x float> addrspace(1)* %out) #0 {
98  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
99  %arrayidx0 = getelementptr inbounds [512 x <4 x float>], [512 x <4 x float>] addrspace(3)* @lds.v4, i32 0, i32 %x.i
100  %val0 = load <4 x float>, <4 x float> addrspace(3)* %arrayidx0, align 8
101  %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %out, i32 %x.i
102  store <4 x float> %val0, <4 x float> addrspace(1)* %out.gep
103  ret void
104}
105
106; CI-LABEL: {{^}}simple_read2_v4f32_superreg:
107; CI: ds_read2_b64 v{{\[}}[[REG_W:[0-9]+]]:[[REG_Z:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1{{$}}
108; CI: v_mov_b32
109; CI: v_mov_b32
110; CI: buffer_store_dwordx4
111; CI: s_endpgm
112define void @simple_read2_v4f32_superreg(<4 x float> addrspace(1)* %out) #0 {
113  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
114  %arrayidx0 = getelementptr inbounds [512 x <4 x float>], [512 x <4 x float>] addrspace(3)* @lds.v4, i32 0, i32 %x.i
115  %val0 = load <4 x float>, <4 x float> addrspace(3)* %arrayidx0
116  %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %out, i32 %x.i
117  store <4 x float> %val0, <4 x float> addrspace(1)* %out.gep
118  ret void
119}
120
121; FIXME: Extra moves shuffling superregister
122; CI-LABEL: {{^}}simple_read2_v8f32_superreg:
123; CI: ds_read2_b64 v{{\[}}[[REG_ELT3:[0-9]+]]:[[REG_ELT7:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1 offset1:3{{$}}
124; CI: v_mov_b32
125; CI: v_mov_b32
126; CI: ds_read2_b64 v{{\[}}[[REG_ELT6:[0-9]+]]:[[REG_ELT5:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:2{{$}}
127; CI: v_mov_b32
128; CI: v_mov_b32
129; CI: buffer_store_dwordx4
130; CI: buffer_store_dwordx4
131; CI: s_endpgm
132define void @simple_read2_v8f32_superreg(<8 x float> addrspace(1)* %out) #0 {
133  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
134  %arrayidx0 = getelementptr inbounds [512 x <8 x float>], [512 x <8 x float>] addrspace(3)* @lds.v8, i32 0, i32 %x.i
135  %val0 = load <8 x float>, <8 x float> addrspace(3)* %arrayidx0
136  %out.gep = getelementptr inbounds <8 x float>, <8 x float> addrspace(1)* %out, i32 %x.i
137  store <8 x float> %val0, <8 x float> addrspace(1)* %out.gep
138  ret void
139}
140
141; FIXME: Extra moves shuffling superregister
142; CI-LABEL: {{^}}simple_read2_v16f32_superreg:
143; CI: ds_read2_b64 v{{\[}}[[REG_ELT11:[0-9]+]]:[[REG_ELT15:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1 offset1:3{{$}}
144; CI: v_mov_b32
145; CI: v_mov_b32
146; CI: ds_read2_b64 v{{\[}}[[REG_ELT14:[0-9]+]]:[[REG_ELT13:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:5 offset1:7{{$}}
147; CI: ds_read2_b64 v{{\[}}[[REG_ELT14:[0-9]+]]:[[REG_ELT13:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:6 offset1:4{{$}}
148; CI: v_mov_b32
149; CI: v_mov_b32
150; CI: ds_read2_b64 v{{\[}}[[REG_ELT12:[0-9]+]]:[[REG_ELT10:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:2{{$}}
151; CI: v_mov_b32
152; CI: v_mov_b32
153
154; CI: s_waitcnt lgkmcnt(0)
155; CI: buffer_store_dwordx4
156; CI: buffer_store_dwordx4
157; CI: buffer_store_dwordx4
158; CI: buffer_store_dwordx4
159; CI: s_endpgm
160define void @simple_read2_v16f32_superreg(<16 x float> addrspace(1)* %out) #0 {
161  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
162  %arrayidx0 = getelementptr inbounds [512 x <16 x float>], [512 x <16 x float>] addrspace(3)* @lds.v16, i32 0, i32 %x.i
163  %val0 = load <16 x float>, <16 x float> addrspace(3)* %arrayidx0
164  %out.gep = getelementptr inbounds <16 x float>, <16 x float> addrspace(1)* %out, i32 %x.i
165  store <16 x float> %val0, <16 x float> addrspace(1)* %out.gep
166  ret void
167}
168
169; Do scalar loads into the super register we need.
170; CI-LABEL: {{^}}simple_read2_v2f32_superreg_scalar_loads_align4:
171; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT0:[0-9]+]]:[[REG_ELT1:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1{{$}}
172; CI-NOT: v_mov
173; CI: buffer_store_dwordx2 v{{\[}}[[REG_ELT0]]:[[REG_ELT1]]{{\]}}
174; CI: s_endpgm
175define void @simple_read2_v2f32_superreg_scalar_loads_align4(<2 x float> addrspace(1)* %out) #0 {
176  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
177  %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
178  %arrayidx1 = getelementptr inbounds float, float addrspace(3)* %arrayidx0, i32 1
179
180  %val0 = load float, float addrspace(3)* %arrayidx0
181  %val1 = load float, float addrspace(3)* %arrayidx1
182
183  %vec.0 = insertelement <2 x float> undef, float %val0, i32 0
184  %vec.1 = insertelement <2 x float> %vec.0, float %val1, i32 1
185
186  %out.gep = getelementptr inbounds <2 x float>, <2 x float> addrspace(1)* %out, i32 %x.i
187  store <2 x float> %vec.1, <2 x float> addrspace(1)* %out.gep
188  ret void
189}
190
191; Do scalar loads into the super register we need.
192; CI-LABEL: {{^}}simple_read2_v4f32_superreg_scalar_loads_align4:
193; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT0:[0-9]+]]:[[REG_ELT1:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1{{$}}
194; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT2:[0-9]+]]:[[REG_ELT3:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
195; CI-NOT: v_mov
196; CI: buffer_store_dwordx4 v{{\[}}[[REG_ELT0]]:[[REG_ELT3]]{{\]}}
197; CI: s_endpgm
198define void @simple_read2_v4f32_superreg_scalar_loads_align4(<4 x float> addrspace(1)* %out) #0 {
199  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
200  %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
201  %arrayidx1 = getelementptr inbounds float, float addrspace(3)* %arrayidx0, i32 1
202  %arrayidx2 = getelementptr inbounds float, float addrspace(3)* %arrayidx0, i32 2
203  %arrayidx3 = getelementptr inbounds float, float addrspace(3)* %arrayidx0, i32 3
204
205  %val0 = load float, float addrspace(3)* %arrayidx0
206  %val1 = load float, float addrspace(3)* %arrayidx1
207  %val2 = load float, float addrspace(3)* %arrayidx2
208  %val3 = load float, float addrspace(3)* %arrayidx3
209
210  %vec.0 = insertelement <4 x float> undef, float %val0, i32 0
211  %vec.1 = insertelement <4 x float> %vec.0, float %val1, i32 1
212  %vec.2 = insertelement <4 x float> %vec.1, float %val2, i32 2
213  %vec.3 = insertelement <4 x float> %vec.2, float %val3, i32 3
214
215  %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %out, i32 %x.i
216  store <4 x float> %vec.3, <4 x float> addrspace(1)* %out.gep
217  ret void
218}
219
220; Function Attrs: nounwind readnone
221declare i32 @llvm.r600.read.tgid.x() #1
222
223; Function Attrs: nounwind readnone
224declare i32 @llvm.r600.read.tgid.y() #1
225
226; Function Attrs: nounwind readnone
227declare i32 @llvm.r600.read.tidig.x() #1
228
229; Function Attrs: nounwind readnone
230declare i32 @llvm.r600.read.tidig.y() #1
231
232; Function Attrs: convergent nounwind
233declare void @llvm.AMDGPU.barrier.local() #2
234
235attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
236attributes #1 = { nounwind readnone }
237attributes #2 = { convergent nounwind }
238