1; RUN: llc -march=amdgcn -mtriple=amdgcn-- -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI,SIVI %s
2; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,SIVI,GFX89 %s
3; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX89 %s
4
5; GCN-LABEL: {{^}}extract_vector_elt_v2i16:
6; GCN: s_load_dword [[VEC:s[0-9]+]]
7; GCN: s_lshr_b32 [[ELT1:s[0-9]+]], [[VEC]], 16
8; GCN-DAG: v_mov_b32_e32 [[VELT0:v[0-9]+]], [[VEC]]
9; GCN-DAG: v_mov_b32_e32 [[VELT1:v[0-9]+]], [[ELT1]]
10; GCN-DAG: buffer_store_short [[VELT0]]
11; GCN-DAG: buffer_store_short [[VELT1]]
12define amdgpu_kernel void @extract_vector_elt_v2i16(i16 addrspace(1)* %out, <2 x i16> addrspace(4)* %vec.ptr) #0 {
13  %vec = load <2 x i16>, <2 x i16> addrspace(4)* %vec.ptr
14  %p0 = extractelement <2 x i16> %vec, i32 0
15  %p1 = extractelement <2 x i16> %vec, i32 1
16  %out1 = getelementptr i16, i16 addrspace(1)* %out, i32 10
17  store i16 %p1, i16 addrspace(1)* %out, align 2
18  store i16 %p0, i16 addrspace(1)* %out1, align 2
19  ret void
20}
21
22; GCN-LABEL: {{^}}extract_vector_elt_v2i16_dynamic_sgpr:
23; GCN: s_load_dword [[IDX:s[0-9]+]]
24; GCN: s_load_dword [[VEC:s[0-9]+]]
25; GCN: s_lshl_b32 [[IDX_SCALED:s[0-9]+]], [[IDX]], 4
26; GCN: s_lshr_b32 [[ELT1:s[0-9]+]], [[VEC]], [[IDX_SCALED]]
27; GCN: v_mov_b32_e32 [[VELT1:v[0-9]+]], [[ELT1]]
28; GCN: buffer_store_short [[VELT1]]
29; GCN: ScratchSize: 0
30define amdgpu_kernel void @extract_vector_elt_v2i16_dynamic_sgpr(i16 addrspace(1)* %out, <2 x i16> addrspace(4)* %vec.ptr, [8 x i32], i32 %idx) #0 {
31  %vec = load <2 x i16>, <2 x i16> addrspace(4)* %vec.ptr
32  %elt = extractelement <2 x i16> %vec, i32 %idx
33  store i16 %elt, i16 addrspace(1)* %out, align 2
34  ret void
35}
36
37; GCN-LABEL: {{^}}extract_vector_elt_v2i16_dynamic_vgpr:
38; GCN-DAG: s_load_dword [[VEC:s[0-9]+]]
39; GCN-DAG: {{flat|buffer|global}}_load_dword [[IDX:v[0-9]+]]
40; GCN: v_lshlrev_b32_e32 [[IDX_SCALED:v[0-9]+]], 4, [[IDX]]
41
42; SI: v_lshr_b32_e32 [[ELT:v[0-9]+]], [[VEC]], [[IDX_SCALED]]
43; VI: v_lshrrev_b32_e64 [[ELT:v[0-9]+]], [[IDX_SCALED]], [[VEC]]
44
45; SI: buffer_store_short [[ELT]]
46; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[ELT]]
47; GCN: ScratchSize: 0{{$}}
48define amdgpu_kernel void @extract_vector_elt_v2i16_dynamic_vgpr(i16 addrspace(1)* %out, <2 x i16> addrspace(4)* %vec.ptr, i32 addrspace(1)* %idx.ptr) #0 {
49  %tid = call i32 @llvm.amdgcn.workitem.id.x()
50  %tid.ext = sext i32 %tid to i64
51  %gep = getelementptr inbounds i32, i32 addrspace(1)* %idx.ptr, i64 %tid.ext
52  %out.gep = getelementptr inbounds i16, i16 addrspace(1)* %out, i64 %tid.ext
53  %idx = load volatile i32, i32 addrspace(1)* %gep
54  %vec = load <2 x i16>, <2 x i16> addrspace(4)* %vec.ptr
55  %elt = extractelement <2 x i16> %vec, i32 %idx
56  store i16 %elt, i16 addrspace(1)* %out.gep, align 2
57  ret void
58}
59
60; GCN-LABEL: {{^}}extract_vector_elt_v3i16:
61; GCN: s_load_dwordx2
62; GCN: s_load_dwordx2
63
64; GCN-NOT: {{buffer|flat|global}}_load
65
66; GCN: buffer_store_short
67; GCN: buffer_store_short
68define amdgpu_kernel void @extract_vector_elt_v3i16(i16 addrspace(1)* %out, <3 x i16> %foo) #0 {
69  %p0 = extractelement <3 x i16> %foo, i32 0
70  %p1 = extractelement <3 x i16> %foo, i32 2
71  %out1 = getelementptr i16, i16 addrspace(1)* %out, i32 1
72  store i16 %p1, i16 addrspace(1)* %out, align 2
73  store i16 %p0, i16 addrspace(1)* %out1, align 2
74  ret void
75}
76
77; GCN-LABEL: {{^}}extract_vector_elt_v4i16:
78; SI: s_load_dwordx2
79; SI: buffer_store_short
80; SI: buffer_store_short
81
82; GFX89-DAG: s_load_dwordx2 s{{\[}}[[LOAD0:[0-9]+]]:[[LOAD1:[0-9]+]]{{\]}}, s[0:1], 0x2c
83; GFX89-DAG: v_mov_b32_e32 [[VLOAD0:v[0-9]+]], s[[LOAD0]]
84; GFX89-DAG: buffer_store_short [[VLOAD0]], off
85; GFX89-DAG: v_mov_b32_e32 [[VLOAD1:v[0-9]+]], s[[LOAD1]]
86; GFX89-DAG: buffer_store_short [[VLOAD1]], off
87define amdgpu_kernel void @extract_vector_elt_v4i16(i16 addrspace(1)* %out, <4 x i16> %foo) #0 {
88  %p0 = extractelement <4 x i16> %foo, i32 0
89  %p1 = extractelement <4 x i16> %foo, i32 2
90  %out1 = getelementptr i16, i16 addrspace(1)* %out, i32 10
91  store volatile i16 %p1, i16 addrspace(1)* %out, align 2
92  store volatile i16 %p0, i16 addrspace(1)* %out1, align 2
93  ret void
94}
95
96; GCN-LABEL: {{^}}dynamic_extract_vector_elt_v3i16:
97; SI: s_load_dword s
98; SI: s_load_dwordx2 s
99; SI: s_load_dwordx2 s
100
101; GFX89-DAG: s_load_dwordx2 s{{\[}}[[LOAD0:[0-9]+]]:[[LOAD1:[0-9]+]]{{\]}}, s[0:1], 0x24
102; GFX89-DAG: s_load_dwordx2 s{{\[}}[[LOAD0:[0-9]+]]:[[LOAD1:[0-9]+]]{{\]}}, s[0:1], 0x4c
103; GFX89-DAG: s_load_dword s{{[0-9]+}}, s[0:1], 0x54
104
105; GCN-NOT: {{buffer|flat|global}}
106
107; SICI: buffer_store_short
108; SICI: buffer_store_short
109; SICI: buffer_store_short
110
111; GFX9-NOT: s_pack_ll_b32_b16
112; GFX9-NOT: s_pack_lh_b32_b16
113
114; GCN-DAG: s_lshl_b32 s{{[0-9]+}}, s{{[0-9]+}}, 4
115; GCN: s_lshr_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s
116; GCN: {{buffer|global}}_store_short
117define amdgpu_kernel void @dynamic_extract_vector_elt_v3i16(i16 addrspace(1)* %out, [8 x i32], <3 x i16> %foo, i32 %idx) #0 {
118  %p0 = extractelement <3 x i16> %foo, i32 %idx
119  %out1 = getelementptr i16, i16 addrspace(1)* %out, i32 1
120  store i16 %p0, i16 addrspace(1)* %out
121  ret void
122}
123
124; GCN-LABEL: {{^}}v_insertelement_v4i16_dynamic_sgpr:
125define amdgpu_kernel void @v_insertelement_v4i16_dynamic_sgpr(i16 addrspace(1)* %out, <4 x i16> addrspace(1)* %in, i32 %idx) #0 {
126  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
127  %tid.ext = sext i32 %tid to i64
128  %in.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i64 %tid.ext
129  %out.gep = getelementptr inbounds i16, i16 addrspace(1)* %out, i64 %tid.ext
130  %vec = load <4 x i16>, <4 x i16> addrspace(1)* %in.gep
131  %vec.extract = extractelement <4 x i16> %vec, i32 %idx
132  store i16 %vec.extract, i16 addrspace(1)* %out.gep
133  ret void
134}
135
136; GCN-LABEL: {{^}}reduce_load_vector_v8i16_extract_01:
137; GCN: s_load_dwordx2 [[PTR:s\[[0-9]+:[0-9]+\]]],
138; GCN-NOT: {{s|buffer|flat|global}}_load_
139; GCN: s_load_dword s{{[0-9]+}}, [[PTR]], 0x0
140; GCN-NOT: {{s|buffer|flat|global}}_load_
141; GCN: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 16
142define amdgpu_kernel void @reduce_load_vector_v8i16_extract_01(<16 x i16> addrspace(4)* %ptr) #0 {
143  %load = load <16 x i16>, <16 x i16> addrspace(4)* %ptr
144  %elt0 = extractelement <16 x i16> %load, i32 0
145  %elt1 = extractelement <16 x i16> %load, i32 1
146  store volatile i16 %elt0, i16 addrspace(1)* undef, align 2
147  store volatile i16 %elt1, i16 addrspace(1)* undef, align 2
148  ret void
149}
150
151; GCN-LABEL: {{^}}reduce_load_vector_v8i16_extract_23:
152; GCN: s_load_dwordx2 [[PTR:s\[[0-9]+:[0-9]+\]]],
153; GCN-NOT: {{s|buffer|flat|global}}_load_
154; GCN: s_load_dword s{{[0-9]+}}, [[PTR]], {{0x1|0x4}}
155; GCN-NOT: {{s|buffer|flat|global}}_load_
156; GCN: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 16
157define amdgpu_kernel void @reduce_load_vector_v8i16_extract_23(<16 x i16> addrspace(4)* %ptr) #0 {
158  %load = load <16 x i16>, <16 x i16> addrspace(4)* %ptr
159  %elt2 = extractelement <16 x i16> %load, i32 2
160  %elt3 = extractelement <16 x i16> %load, i32 3
161  store volatile i16 %elt2, i16 addrspace(1)* undef, align 2
162  store volatile i16 %elt3, i16 addrspace(1)* undef, align 2
163  ret void
164}
165
166declare i32 @llvm.amdgcn.workitem.id.x() #1
167
168attributes #0 = { nounwind }
169attributes #1 = { nounwind readnone }
170