1; RUN: opt -loop-vectorize -force-vector-width=4 -enable-vplan-native-path -S %s | FileCheck %s
2
3; Vectorize explict marked outer loop using vplan native path. Inner loop
4; contains simple double add reduction. IR is compiled and modified by hand
5; from following C code:
6; void inner_loop_reduction(const double* restrict in_a, const double* restrict in_b, double* restrict out)
7; {
8;     #pragma clang loop vectorize(enable)
9;     for (int i = 0; i < 1000; ++i) {
10;         double a = in_a[i];
11;         double b = in_b[i];
12;         for (int j = 0; j < 10000; ++j) {
13;             a = a + b;
14;         }
15;         out[i] = a;
16;     }
17; }
18define void @inner_loop_reduction(double* noalias nocapture readonly %a.in, double* noalias nocapture readonly %b.in, double* noalias nocapture %c.out) {
19; CHECK-LABEL: @inner_loop_reduction(
20
21; CHECK: vector.body:
22; CHECK-NEXT: %[[FOR1_INDEX:.*]] = phi i64 [ 0, %[[LABEL_PR:.*]] ], [ %{{.*}}, %[[LABEL_FOR1_LATCH:.*]] ]
23; CHECK: %[[VEC_INDEX:.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[LABEL_PR]] ], [ %{{.*}}, %[[LABEL_FOR1_LATCH]] ]
24; CHECK-NEXT: %[[A_PTR:.*]] = getelementptr inbounds double, double* %a.in, <4 x i64> %[[VEC_INDEX]]
25; CHECK-NEXT: %[[MASKED_GATHER1:.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> %[[A_PTR]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x double> undef)
26; CHECK-NEXT: %[[B_PTR:.*]] = getelementptr inbounds double, double* %b.in, <4 x i64> %[[VEC_INDEX]]
27; CHECK-NEXT: %[[MASKED_GATHER2:.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> %[[B_PTR]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x double> undef)
28; CHECK-NEXT: br label %[[FOR2_HEADER:.*]]
29
30; CHECK: [[FOR2_HEADER]]:
31; CHECK-NEXT: %[[FOR2_INDEX:.*]] = phi <4 x i32> [ %[[FOR2_INDEX_NEXT:.*]], %[[FOR2_HEADER]] ], [ zeroinitializer, %vector.body ]
32; CHECK-NEXT: %[[REDUCTION:.*]] = phi <4 x double> [ %[[REDUCTION_NEXT:.*]], %[[FOR2_HEADER]] ], [ %[[MASKED_GATHER1]], %vector.body ]
33; CHECK-NEXT: %[[REDUCTION_NEXT]] = fadd <4 x double> %[[MASKED_GATHER2]], %[[REDUCTION]]
34; CHECK-NEXT: %[[FOR2_INDEX_NEXT]] = add nuw nsw <4 x i32> %[[FOR2_INDEX]], <i32 1, i32 1, i32 1, i32 1>
35; CHECK-NEXT: %[[VEC_PTR:.*]] = icmp eq <4 x i32> %[[FOR2_INDEX_NEXT]], <i32 10000, i32 10000, i32 10000, i32 10000>
36; CHECK-NEXT: %[[EXIT_COND:.*]] = extractelement <4 x i1> %[[VEC_PTR]], i32 0
37; CHECK-NEXT: br i1 %[[EXIT_COND]], label %[[FOR1_LATCH:.*]], label %{{.*}}
38
39; CHECK: [[FOR1_LATCH]]:
40; CHECK-NEXT: %[[REDUCTION:.*]] = phi <4 x double> [ %[[REDUCTION_NEXT]], %[[FOR2_HEADER]] ]
41; CHECK-NEXT: %[[C_PTR:.*]] = getelementptr inbounds double, double* %c.out, <4 x i64> %[[VEC_INDEX]]
42; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> %[[REDUCTION]], <4 x double*> %[[C_PTR]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
43; CHECK-NEXT: %[[VEC_INDEX_NEXT:.*]] = add nuw nsw <4 x i64> %[[VEC_INDEX]], <i64 1, i64 1, i64 1, i64 1>
44; CHECK-NEXT: %[[VEC_PTR:.*]] = icmp eq <4 x i64> %[[VEC_INDEX_NEXT]], <i64 1000, i64 1000, i64 1000, i64 1000>
45; CHECK-NEXT: %{{.*}} = extractelement <4 x i1> %[[VEC_PTR]], i32 0
46; CHECK-NEXT: %[[FOR1_INDEX_NEXT:.*]] = add i64 %[[FOR1_INDEX]], 4
47; CHECK-NEXT: %{{.*}} = add <4 x i64> %[[VEC_INDEX]], <i64 4, i64 4, i64 4, i64 4>
48; CHECK-NEXT: %[[EXIT_COND:.*]] = icmp eq i64 %[[FOR1_INDEX_NEXT]], 1000
49; CHECK-NEXT: br i1 %[[EXIT_COND]], label %{{.*}}, label %vector.body
50
51entry:
52  br label %for1.header
53
54for1.header:                                              ; preds = %entry
55  %indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ]
56  %a.ptr = getelementptr inbounds double, double* %a.in, i64 %indvar1
57  %a = load double, double* %a.ptr, align 8
58  %b.ptr = getelementptr inbounds double, double* %b.in, i64 %indvar1
59  %b = load double, double* %b.ptr, align 8
60  br label %for2.header
61
62for2.header:                                              ; preds = %for1.header, %for2.header
63  %indvar2 = phi i32 [ 0, %for1.header ], [ %indvar21, %for2.header ]
64  %a.reduction = phi double [ %a, %for1.header ], [ %a.reduction1, %for2.header ]
65  %a.reduction1 = fadd double %b, %a.reduction
66  %indvar21 = add nuw nsw i32 %indvar2, 1
67  %for2.cond = icmp eq i32 %indvar21, 10000
68  br i1 %for2.cond, label %for1.latch, label %for2.header
69
70for1.latch:                                               ; preds = %for2.header
71  %c.ptr = getelementptr inbounds double, double* %c.out, i64 %indvar1
72  store double %a.reduction1, double* %c.ptr, align 8
73  %indvar11 = add nuw nsw i64 %indvar1, 1
74  %for1.cond = icmp eq i64 %indvar11, 1000
75  br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0
76
77exit:                                                    ; preds = %for1.latch
78  ret void
79}
80
81!0 = distinct !{!0, !1}
82!1 = !{!"llvm.loop.vectorize.enable", i1 true}
83