1; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve.fp -loop-vectorize -tail-predication=enabled -S < %s | \
2; RUN:  FileCheck %s
3
4; Test that ARMTTIImpl::preferPredicateOverEpilogue triggers tail-folding.
5
6define dso_local void @f1(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32 %N) {
7; CHECK-LABEL: f1(
8; CHECK:       entry:
9; CHECK:       @llvm.get.active.lane.mask
10; CHECK:       }
11entry:
12  %cmp8 = icmp sgt i32 %N, 0
13  br i1 %cmp8, label %for.body.preheader, label %for.cond.cleanup
14
15for.body.preheader:                               ; preds = %entry
16  br label %for.body
17
18for.cond.cleanup.loopexit:                        ; preds = %for.body
19  br label %for.cond.cleanup
20
21for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
22  ret void
23
24for.body:                                         ; preds = %for.body.preheader, %for.body
25  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
26  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.09
27  %0 = load i32, i32* %arrayidx, align 4
28  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.09
29  %1 = load i32, i32* %arrayidx1, align 4
30  %add = add nsw i32 %1, %0
31  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.09
32  store i32 %add, i32* %arrayidx2, align 4
33  %inc = add nuw nsw i32 %i.09, 1
34  %exitcond.not = icmp eq i32 %inc, %N
35  br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body
36}
37
38define dso_local void @f32_reduction(float* nocapture readonly %Input, i32 %N, float* nocapture %Output) {
39; CHECK-LABEL: f32_reduction(
40; CHECK:       vector.body:
41; CHECK:       @llvm.masked.load
42; CHECK:       br i1 %{{.*}}, label {{.*}}, label %vector.body
43entry:
44  %cmp6 = icmp eq i32 %N, 0
45  br i1 %cmp6, label %while.end, label %while.body.preheader
46
47while.body.preheader:                             ; preds = %entry
48  br label %while.body
49
50while.body:                                       ; preds = %while.body.preheader, %while.body
51  %blkCnt.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
52  %sum.08 = phi float [ %add, %while.body ], [ 0.000000e+00, %while.body.preheader ]
53  %Input.addr.07 = phi float* [ %incdec.ptr, %while.body ], [ %Input, %while.body.preheader ]
54  %incdec.ptr = getelementptr inbounds float, float* %Input.addr.07, i32 1
55  %0 = load float, float* %Input.addr.07, align 4
56  %add = fadd fast float %0, %sum.08
57  %dec = add i32 %blkCnt.09, -1
58  %cmp = icmp eq i32 %dec, 0
59  br i1 %cmp, label %while.end.loopexit, label %while.body
60
61while.end.loopexit:                               ; preds = %while.body
62  %add.lcssa = phi float [ %add, %while.body ]
63  br label %while.end
64
65while.end:                                        ; preds = %while.end.loopexit, %entry
66  %sum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add.lcssa, %while.end.loopexit ]
67  %conv = uitofp i32 %N to float
68  %div = fdiv fast float %sum.0.lcssa, %conv
69  store float %div, float* %Output, align 4
70  ret void
71}
72
73define dso_local void @f16_reduction(half* nocapture readonly %Input, i32 %N, half* nocapture %Output) {
74; CHECK-LABEL: f16_reduction(
75; CHECK:       vector.body:
76; CHECK:       @llvm.masked.load
77; CHECK:       br i1 %{{.*}}, label {{.*}}, label %vector.body
78entry:
79  %cmp6 = icmp eq i32 %N, 0
80  br i1 %cmp6, label %while.end, label %while.body.preheader
81
82while.body.preheader:                             ; preds = %entry
83  br label %while.body
84
85while.body:                                       ; preds = %while.body.preheader, %while.body
86  %blkCnt.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
87  %sum.08 = phi half [ %add, %while.body ], [ 0.000000e+00, %while.body.preheader ]
88  %Input.addr.07 = phi half* [ %incdec.ptr, %while.body ], [ %Input, %while.body.preheader ]
89  %incdec.ptr = getelementptr inbounds half, half* %Input.addr.07, i32 1
90  %0 = load half, half* %Input.addr.07, align 2
91  %add = fadd fast half %0, %sum.08
92  %dec = add i32 %blkCnt.09, -1
93  %cmp = icmp eq i32 %dec, 0
94  br i1 %cmp, label %while.end.loopexit, label %while.body
95
96while.end.loopexit:                               ; preds = %while.body
97  %add.lcssa = phi half [ %add, %while.body ]
98  br label %while.end
99
100while.end:                                        ; preds = %while.end.loopexit, %entry
101  %sum.0.lcssa = phi half [ 0.000000e+00, %entry ], [ %add.lcssa, %while.end.loopexit ]
102  %conv = uitofp i32 %N to half
103  %div = fdiv fast half %sum.0.lcssa, %conv
104  store half %div, half* %Output, align 2
105  ret void
106}
107
108define dso_local void @mixed_f32_i32_reduction(float* nocapture readonly %fInput, i32* nocapture readonly %iInput, i32 %N, float* nocapture %fOutput, i32* nocapture %iOutput) {
109; CHECK-LABEL: mixed_f32_i32_reduction(
110; CHECK:       vector.body:
111; CHECK:       @llvm.masked.load
112; CHECK:       br i1 %{{.*}}, label {{.*}}, label %vector.body
113entry:
114  %cmp15 = icmp eq i32 %N, 0
115  br i1 %cmp15, label %while.end, label %while.body.preheader
116
117while.body.preheader:
118  br label %while.body
119
120while.body:
121  %blkCnt.020 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
122  %isum.019 = phi i32 [ %add2, %while.body ], [ 0, %while.body.preheader ]
123  %fsum.018 = phi float [ %add, %while.body ], [ 0.000000e+00, %while.body.preheader ]
124  %fInput.addr.017 = phi float* [ %incdec.ptr, %while.body ], [ %fInput, %while.body.preheader ]
125  %iInput.addr.016 = phi i32* [ %incdec.ptr1, %while.body ], [ %iInput, %while.body.preheader ]
126  %incdec.ptr = getelementptr inbounds float, float* %fInput.addr.017, i32 1
127  %incdec.ptr1 = getelementptr inbounds i32, i32* %iInput.addr.016, i32 1
128  %0 = load i32, i32* %iInput.addr.016, align 4
129  %add2 = add nsw i32 %0, %isum.019
130  %1 = load float, float* %fInput.addr.017, align 4
131  %add = fadd fast float %1, %fsum.018
132  %dec = add i32 %blkCnt.020, -1
133  %cmp = icmp eq i32 %dec, 0
134  br i1 %cmp, label %while.end.loopexit, label %while.body
135
136while.end.loopexit:
137  %add.lcssa = phi float [ %add, %while.body ]
138  %add2.lcssa = phi i32 [ %add2, %while.body ]
139  %phitmp = sitofp i32 %add2.lcssa to float
140  br label %while.end
141
142while.end:
143  %fsum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add.lcssa, %while.end.loopexit ]
144  %isum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %phitmp, %while.end.loopexit ]
145  %conv = uitofp i32 %N to float
146  %div = fdiv fast float %fsum.0.lcssa, %conv
147  store float %div, float* %fOutput, align 4
148  %div5 = fdiv fast float %isum.0.lcssa, %conv
149  %conv6 = fptosi float %div5 to i32
150  store i32 %conv6, i32* %iOutput, align 4
151  ret void
152}
153
154define dso_local i32 @i32_mul_reduction(i32* noalias nocapture readonly %B, i32 %N) {
155; CHECK-LABEL: i32_mul_reduction(
156; CHECK:       vector.body:
157; CHECK:       @llvm.masked.load
158; CHECK:       br i1 %{{.*}}, label {{.*}}, label %vector.body
159entry:
160  %cmp6 = icmp sgt i32 %N, 0
161  br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
162
163for.body.preheader:
164  br label %for.body
165
166for.cond.cleanup.loopexit:
167  %mul.lcssa = phi i32 [ %mul, %for.body ]
168  br label %for.cond.cleanup
169
170for.cond.cleanup:
171  %S.0.lcssa = phi i32 [ 1, %entry ], [ %mul.lcssa, %for.cond.cleanup.loopexit ]
172  ret i32 %S.0.lcssa
173
174for.body:
175  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
176  %S.07 = phi i32 [ %mul, %for.body ], [ 1, %for.body.preheader ]
177  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.08
178  %0 = load i32, i32* %arrayidx, align 4
179  %mul = mul nsw i32 %0, %S.07
180  %inc = add nuw nsw i32 %i.08, 1
181  %exitcond = icmp eq i32 %inc, %N
182  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
183}
184
185define dso_local i32 @i32_or_reduction(i32* noalias nocapture readonly %B, i32 %N) {
186; CHECK-LABEL: i32_or_reduction(
187; CHECK:       vector.body:
188; CHECK:       @llvm.masked.load
189; CHECK:       br i1 %{{.*}}, label {{.*}}, label %vector.body
190entry:
191  %cmp6 = icmp sgt i32 %N, 0
192  br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
193
194for.body.preheader:                               ; preds = %entry
195  br label %for.body
196
197for.cond.cleanup.loopexit:                        ; preds = %for.body
198  %or.lcssa = phi i32 [ %or, %for.body ]
199  br label %for.cond.cleanup
200
201for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
202  %S.0.lcssa = phi i32 [ 1, %entry ], [ %or.lcssa, %for.cond.cleanup.loopexit ]
203  ret i32 %S.0.lcssa
204
205for.body:                                         ; preds = %for.body.preheader, %for.body
206  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
207  %S.07 = phi i32 [ %or, %for.body ], [ 1, %for.body.preheader ]
208  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.08
209  %0 = load i32, i32* %arrayidx, align 4
210  %or = or i32 %0, %S.07
211  %inc = add nuw nsw i32 %i.08, 1
212  %exitcond = icmp eq i32 %inc, %N
213  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
214}
215
216define dso_local i32 @i32_and_reduction(i32* noalias nocapture readonly %A, i32 %N, i32 %S) {
217; CHECK-LABEL: i32_and_reduction(
218; CHECK:       vector.body:
219; CHECK:       @llvm.masked.load
220; CHECK:       br i1 %{{.*}}, label {{.*}}, label %vector.body
221entry:
222  %cmp5 = icmp sgt i32 %N, 0
223  br i1 %cmp5, label %for.body.preheader, label %for.cond.cleanup
224
225for.body.preheader:                               ; preds = %entry
226  br label %for.body
227
228for.cond.cleanup.loopexit:                        ; preds = %for.body
229  %and.lcssa = phi i32 [ %and, %for.body ]
230  br label %for.cond.cleanup
231
232for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
233  %S.addr.0.lcssa = phi i32 [ %S, %entry ], [ %and.lcssa, %for.cond.cleanup.loopexit ]
234  ret i32 %S.addr.0.lcssa
235
236for.body:                                         ; preds = %for.body.preheader, %for.body
237  %i.07 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
238  %S.addr.06 = phi i32 [ %and, %for.body ], [ %S, %for.body.preheader ]
239  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.07
240  %0 = load i32, i32* %arrayidx, align 4
241  %and = and i32 %0, %S.addr.06
242  %inc = add nuw nsw i32 %i.07, 1
243  %exitcond = icmp eq i32 %inc, %N
244  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
245}
246