1; RUN: llc < %s -march=nvptx64 -mcpu=sm_30 | FileCheck %s
2
3declare i32 @llvm.nvvm.shfl.down.i32(i32, i32, i32)
4declare float @llvm.nvvm.shfl.down.f32(float, i32, i32)
5declare i32 @llvm.nvvm.shfl.up.i32(i32, i32, i32)
6declare float @llvm.nvvm.shfl.up.f32(float, i32, i32)
7declare i32 @llvm.nvvm.shfl.bfly.i32(i32, i32, i32)
8declare float @llvm.nvvm.shfl.bfly.f32(float, i32, i32)
9declare i32 @llvm.nvvm.shfl.idx.i32(i32, i32, i32)
10declare float @llvm.nvvm.shfl.idx.f32(float, i32, i32)
11
12; Try all four permutations of register and immediate parameters with
13; shfl.down.
14
15; CHECK-LABEL: .func{{.*}}shfl.down1
16define i32 @shfl.down1(i32 %in) {
17  ; CHECK: ld.param.u32 [[IN:%r[0-9]+]]
18  ; CHECK: shfl.down.b32 [[OUT:%r[0-9]+]], [[IN]], 1, 2;
19  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
20  %val = call i32 @llvm.nvvm.shfl.down.i32(i32 %in, i32 1, i32 2)
21  ret i32 %val
22}
23
24; CHECK-LABEL: .func{{.*}}shfl.down2
25define i32 @shfl.down2(i32 %in, i32 %width) {
26  ; CHECK: ld.param.u32 [[IN1:%r[0-9]+]]
27  ; CHECK: ld.param.u32 [[IN2:%r[0-9]+]]
28  ; CHECK: shfl.down.{{.}}32 %r{{[0-9]+}}, [[IN1]], [[IN2]], 3;
29  %val = call i32 @llvm.nvvm.shfl.down.i32(i32 %in, i32 %width, i32 3)
30  ret i32 %val
31}
32
33; CHECK-LABEL: .func{{.*}}shfl.down3
34define i32 @shfl.down3(i32 %in, i32 %mask) {
35  ; CHECK: ld.param.u32 [[IN1:%r[0-9]+]]
36  ; CHECK: ld.param.u32 [[IN2:%r[0-9]+]]
37  ; CHECK: shfl.down.{{.}}32 %r{{[0-9]+}}, [[IN1]], 4, [[IN2]];
38  %val = call i32 @llvm.nvvm.shfl.down.i32(i32 %in, i32 4, i32 %mask)
39  ret i32 %val
40}
41
42; CHECK-LABEL: .func{{.*}}shfl.down4
43define i32 @shfl.down4(i32 %in, i32 %width, i32 %mask) {
44  ; CHECK: ld.param.u32 [[IN1:%r[0-9]+]]
45  ; CHECK: ld.param.u32 [[IN2:%r[0-9]+]]
46  ; CHECK: ld.param.u32 [[IN3:%r[0-9]+]]
47  ; CHECK: shfl.down.{{.}}32 %r{{[0-9]+}}, [[IN1]], [[IN2]], [[IN3]];
48  %val = call i32 @llvm.nvvm.shfl.down.i32(i32 %in, i32 %width, i32 %mask)
49  ret i32 %val
50}
51
52; Try shfl.down with floating-point params.
53; CHECK-LABEL: .func{{.*}}shfl.down.float
54define float @shfl.down.float(float %in) {
55  ; CHECK: ld.param.f32 [[IN:%f[0-9]+]]
56  ; CHECK: shfl.down.b32 [[OUT:%f[0-9]+]], [[IN]], 5, 6;
57  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
58  %out = call float @llvm.nvvm.shfl.down.f32(float %in, i32 5, i32 6)
59  ret float %out
60}
61
62; Try the rest of the shfl modes.  Hopefully they're declared in such a way
63; that if shfl.down works correctly, they also work correctly.
64define void @shfl.rest(i32 %in_i32, float %in_float, i32* %out_i32, float* %out_float) {
65  ; CHECK: shfl.up.b32 %r{{[0-9]+}}, %r{{[0-9]+}}, 1, 2;
66  %up_i32 = call i32 @llvm.nvvm.shfl.up.i32(i32 %in_i32, i32 1, i32 2)
67  store i32 %up_i32, i32* %out_i32
68
69  ; CHECK: shfl.up.b32 %f{{[0-9]+}}, %f{{[0-9]+}}, 3, 4;
70  %up_float = call float @llvm.nvvm.shfl.up.f32(float %in_float, i32 3, i32 4)
71  store float %up_float, float* %out_float
72
73  ; CHECK: shfl.bfly.b32 %r{{[0-9]+}}, %r{{[0-9]+}}, 5, 6;
74  %bfly_i32 = call i32 @llvm.nvvm.shfl.bfly.i32(i32 %in_i32, i32 5, i32 6)
75  store i32 %bfly_i32, i32* %out_i32
76
77  ; CHECK: shfl.bfly.b32 %f{{[0-9]+}}, %f{{[0-9]+}}, 7, 8;
78  %bfly_float = call float @llvm.nvvm.shfl.bfly.f32(float %in_float, i32 7, i32 8)
79  store float %bfly_float, float* %out_float
80
81  ; CHECK: shfl.idx.b32 %r{{[0-9]+}}, %r{{[0-9]+}}, 9, 10;
82  %idx_i32 = call i32 @llvm.nvvm.shfl.idx.i32(i32 %in_i32, i32 9, i32 10)
83  store i32 %idx_i32, i32* %out_i32
84
85  ; CHECK: shfl.idx.b32 %f{{[0-9]+}}, %f{{[0-9]+}}, 11, 12;
86  %idx_float = call float @llvm.nvvm.shfl.idx.f32(float %in_float, i32 11, i32 12)
87  store float %idx_float, float* %out_float
88
89  ret void
90}
91