1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2
3
4; RUN: opt --arm-mve-gather-scatter-lowering -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp %s -S -o 2>/dev/null - | FileCheck %s
5
6define arm_aapcs_vfpcc void @push_out_add_sub_block(i32* noalias nocapture readonly %data, i32* noalias nocapture %dst, i32 %n.vec) {
7; CHECK-LABEL: @push_out_add_sub_block(
8; CHECK-NEXT:  vector.ph:
9; CHECK-NEXT:    [[IND_END:%.*]] = shl i32 [[N_VEC:%.*]], 1
10; CHECK-NEXT:    [[PUSHEDOUTADD:%.*]] = add <4 x i32> <i32 0, i32 2, i32 4, i32 6>, <i32 6, i32 6, i32 6, i32 6>
11; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
12; CHECK:       vector.body:
13; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY_END:%.*]] ]
14; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ [[PUSHEDOUTADD]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY_END]] ]
15; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX]], 50
16; CHECK-NEXT:    br i1 [[TMP0]], label [[LOWER_BLOCK:%.*]], label [[END:%.*]]
17; CHECK:       lower.block:
18; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i32.v4i32(i32* [[DATA:%.*]], <4 x i32> [[VEC_IND]], i32 32, i32 2, i32 1)
19; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[INDEX]]
20; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
21; CHECK-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP3]], align 4
22; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
23; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 8, i32 8, i32 8, i32 8>
24; CHECK-NEXT:    br label [[VECTOR_BODY_END]]
25; CHECK:       vector.body.end:
26; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
27; CHECK-NEXT:    br i1 [[TMP4]], label [[END]], label [[VECTOR_BODY]]
28; CHECK:       end:
29; CHECK-NEXT:    ret void
30;
31
32vector.ph:
33  %ind.end = shl i32 %n.vec, 1
34  br label %vector.body
35
36vector.body:                                      ; preds = %vector.body, %vector.ph
37  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body.end ]
38  %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body.end ]
39  %0 = icmp eq i32 %index, 50
40  br i1 %0, label %lower.block, label %end
41
42lower.block:                             ; preds = %vector.body
43  %1 = add <4 x i32> %vec.ind, <i32 6, i32 6, i32 6, i32 6>
44  %2 = getelementptr inbounds i32, i32* %data, <4 x i32> %1
45  %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
46  %3 = getelementptr inbounds i32, i32* %dst, i32 %index
47  %4 = bitcast i32* %3 to <4 x i32>*
48  store <4 x i32> %wide.masked.gather, <4 x i32>* %4, align 4
49  %index.next = add i32 %index, 4
50  %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>
51  br label %vector.body.end
52
53vector.body.end:                             ; preds = %lower.block
54  %5 = icmp eq i32 %index.next, %n.vec
55  br i1 %5, label %end, label %vector.body
56
57end:
58  ret void;
59}
60
61define arm_aapcs_vfpcc void @push_out_mul_sub_block(i32* noalias nocapture readonly %data, i32* noalias nocapture %dst, i32 %n.vec) {
62; CHECK-LABEL: @push_out_mul_sub_block(
63; CHECK-NEXT:  vector.ph:
64; CHECK-NEXT:    [[IND_END:%.*]] = shl i32 [[N_VEC:%.*]], 1
65; CHECK-NEXT:    [[PUSHEDOUTMUL:%.*]] = mul <4 x i32> <i32 0, i32 2, i32 4, i32 6>, <i32 3, i32 3, i32 3, i32 3>
66; CHECK-NEXT:    [[PRODUCT:%.*]] = mul <4 x i32> <i32 8, i32 8, i32 8, i32 8>, <i32 3, i32 3, i32 3, i32 3>
67; CHECK-NEXT:    [[PUSHEDOUTADD:%.*]] = add <4 x i32> [[PUSHEDOUTMUL]], <i32 6, i32 6, i32 6, i32 6>
68; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
69; CHECK:       vector.body:
70; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY_END:%.*]] ]
71; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ [[PUSHEDOUTADD]], [[VECTOR_PH]] ], [ [[INCREMENTPUSHEDOUTMUL:%.*]], [[VECTOR_BODY_END]] ]
72; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX]], 50
73; CHECK-NEXT:    br i1 [[TMP0]], label [[LOWER_BLOCK:%.*]], label [[END:%.*]]
74; CHECK:       lower.block:
75; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i32.v4i32(i32* [[DATA:%.*]], <4 x i32> [[VEC_IND]], i32 32, i32 2, i32 1)
76; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[INDEX]]
77; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
78; CHECK-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP3]], align 4
79; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
80; CHECK-NEXT:    br label [[VECTOR_BODY_END]]
81; CHECK:       vector.body.end:
82; CHECK-NEXT:    [[INCREMENTPUSHEDOUTMUL]] = add <4 x i32> [[VEC_IND]], [[PRODUCT]]
83; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
84; CHECK-NEXT:    br i1 [[TMP4]], label [[END]], label [[VECTOR_BODY]]
85; CHECK:       end:
86; CHECK-NEXT:    ret void
87;
88
89vector.ph:
90  %ind.end = shl i32 %n.vec, 1
91  br label %vector.body
92
93vector.body:                                      ; preds = %vector.body, %vector.ph
94  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body.end ]
95  %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body.end ]
96  %0 = icmp eq i32 %index, 50
97  br i1 %0, label %lower.block, label %end
98
99lower.block:                             ; preds = %vector.body
100  %1 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>
101  %2 = add <4 x i32> %1, <i32 6, i32 6, i32 6, i32 6>
102  %3 = getelementptr inbounds i32, i32* %data, <4 x i32> %2
103  %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %3, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
104  %4 = getelementptr inbounds i32, i32* %dst, i32 %index
105  %5 = bitcast i32* %4 to <4 x i32>*
106  store <4 x i32> %wide.masked.gather, <4 x i32>* %5, align 4
107  %index.next = add i32 %index, 4
108  %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>
109  br label %vector.body.end
110
111vector.body.end:                             ; preds = %lower.block
112  %6 = icmp eq i32 %index.next, %n.vec
113  br i1 %6, label %end, label %vector.body
114
115end:
116  ret void;
117}
118
119
120define arm_aapcs_vfpcc void @push_out_mul_sub_loop(i32* noalias nocapture readonly %data, i32* noalias nocapture %dst, i32 %n.vec) {
121; CHECK-LABEL: @push_out_mul_sub_loop(
122; CHECK-NEXT:  vector.ph:
123; CHECK-NEXT:    [[IND_END:%.*]] = shl i32 [[N_VEC:%.*]], 2
124; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
125; CHECK:       vector.body:
126; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY_END:%.*]] ]
127; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY_END]] ]
128; CHECK-NEXT:    br label [[VECTOR_2_PH:%.*]]
129; CHECK:       vector.2.ph:
130; CHECK-NEXT:    br label [[VECTOR_2_BODY:%.*]]
131; CHECK:       vector.2.body:
132; CHECK-NEXT:    [[TMP0:%.*]] = mul <4 x i32> [[VEC_IND]], <i32 3, i32 3, i32 3, i32 3>
133; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[TMP0]], <i32 6, i32 6, i32 6, i32 6>
134; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i32.v4i32(i32* [[DATA:%.*]], <4 x i32> [[TMP1]], i32 32, i32 2, i32 1)
135; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[INDEX]]
136; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
137; CHECK-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* [[TMP4]], align 4
138; CHECK-NEXT:    br label [[VECTOR_2_BODY_END:%.*]]
139; CHECK:       vector.2.body.end:
140; CHECK-NEXT:    [[INDEX_2_NEXT:%.*]] = add i32 [[INDEX]], 4
141; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_2_NEXT]], 15
142; CHECK-NEXT:    br i1 [[TMP5]], label [[VECTOR_BODY_END]], label [[VECTOR_2_BODY]]
143; CHECK:       vector.body.end:
144; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
145; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 8, i32 8, i32 8, i32 8>
146; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
147; CHECK-NEXT:    br i1 [[TMP6]], label [[END:%.*]], label [[VECTOR_BODY]]
148; CHECK:       end:
149; CHECK-NEXT:    ret void
150;
151
152vector.ph:
153  %ind.end = shl i32 %n.vec, 2
154  br label %vector.body
155
156vector.body:                                      ; preds = %vector.body, %vector.ph
157  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body.end ]
158  %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body.end ]
159  br label %vector.2.ph
160
161vector.2.ph:
162  br label %vector.2.body
163
164vector.2.body:                             ; preds = %vector.body
165  %index.2 = phi i32 [ 0, %vector.2.ph ], [ %index.2.next, %vector.2.body.end ]
166  %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>
167  %1 = add <4 x i32> %0, <i32 6, i32 6, i32 6, i32 6>
168  %2 = getelementptr inbounds i32, i32* %data, <4 x i32> %1
169  %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
170  %3 = getelementptr inbounds i32, i32* %dst, i32 %index
171  %4 = bitcast i32* %3 to <4 x i32>*
172  store <4 x i32> %wide.masked.gather, <4 x i32>* %4, align 4
173  br label %vector.2.body.end
174
175vector.2.body.end:                             ; preds = %lower.block
176  %index.2.next = add i32 %index, 4
177  %5 = icmp eq i32 %index.2.next, 15
178  br i1 %5, label %vector.body.end, label %vector.2.body
179
180vector.body.end:                             ; preds = %lower.block
181  %index.next = add i32 %index, 4
182  %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>
183  %6 = icmp eq i32 %index.next, %n.vec
184  br i1 %6, label %end, label %vector.body
185
186end:
187  ret void;
188}
189
190declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
191