1 //===- llvm/unittest/Transforms/Vectorize/VPlanSlpTest.cpp ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "../lib/Transforms/Vectorize/VPlan.h"
10 #include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h"
11 #include "VPlanTestBase.h"
12 #include "llvm/Analysis/TargetLibraryInfo.h"
13 #include "llvm/Analysis/VectorUtils.h"
14 #include "gtest/gtest.h"
15
16 namespace llvm {
17 namespace {
18
19 class VPlanSlpTest : public VPlanTestBase {
20 protected:
21 TargetLibraryInfoImpl TLII;
22 TargetLibraryInfo TLI;
23 DataLayout DL;
24
25 std::unique_ptr<AssumptionCache> AC;
26 std::unique_ptr<ScalarEvolution> SE;
27 std::unique_ptr<AAResults> AARes;
28 std::unique_ptr<BasicAAResult> BasicAA;
29 std::unique_ptr<LoopAccessInfo> LAI;
30 std::unique_ptr<PredicatedScalarEvolution> PSE;
31 std::unique_ptr<InterleavedAccessInfo> IAI;
32
VPlanSlpTest()33 VPlanSlpTest()
34 : TLII(), TLI(TLII),
35 DL("e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-"
36 "f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:"
37 "16:32:64-S128") {}
38
getInterleavedAccessInfo(Function & F,Loop * L,VPlan & Plan)39 VPInterleavedAccessInfo getInterleavedAccessInfo(Function &F, Loop *L,
40 VPlan &Plan) {
41 AC.reset(new AssumptionCache(F));
42 SE.reset(new ScalarEvolution(F, TLI, *AC, *DT, *LI));
43 BasicAA.reset(new BasicAAResult(DL, F, TLI, *AC, &*DT, &*LI));
44 AARes.reset(new AAResults(TLI));
45 AARes->addAAResult(*BasicAA);
46 PSE.reset(new PredicatedScalarEvolution(*SE, *L));
47 LAI.reset(new LoopAccessInfo(L, &*SE, &TLI, &*AARes, &*DT, &*LI));
48 IAI.reset(new InterleavedAccessInfo(*PSE, L, &*DT, &*LI, &*LAI));
49 IAI->analyzeInterleaving(false);
50 return {Plan, *IAI};
51 }
52 };
53
TEST_F(VPlanSlpTest,testSlpSimple_2)54 TEST_F(VPlanSlpTest, testSlpSimple_2) {
55 const char *ModuleString =
56 "%struct.Test = type { i32, i32 }\n"
57 "%struct.Test3 = type { i32, i32, i32 }\n"
58 "%struct.Test4xi8 = type { i8, i8, i8 }\n"
59 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
60 "nocapture readonly %B, %struct.Test* nocapture %C) {\n"
61 "entry:\n"
62 " br label %for.body\n"
63 "for.body: ; preds = %for.body, "
64 "%entry\n"
65 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
66 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
67 "%indvars.iv, i32 0\n"
68 " %vA0 = load i32, i32* %A0, align 4\n"
69 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
70 "%indvars.iv, i32 0\n"
71 " %vB0 = load i32, i32* %B0, align 4\n"
72 " %add0 = add nsw i32 %vA0, %vB0\n"
73 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
74 "%indvars.iv, i32 1\n"
75 " %vA1 = load i32, i32* %A1, align 4\n"
76 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
77 "%indvars.iv, i32 1\n"
78 " %vB1 = load i32, i32* %B1, align 4\n"
79 " %add1 = add nsw i32 %vA1, %vB1\n"
80 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
81 "%indvars.iv, i32 0\n"
82 " store i32 %add0, i32* %C0, align 4\n"
83 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
84 "%indvars.iv, i32 1\n"
85 " store i32 %add1, i32* %C1, align 4\n"
86 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
87 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
88 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
89 "for.cond.cleanup: ; preds = %for.body\n"
90 " ret void\n"
91 "}\n";
92
93 Module &M = parseModule(ModuleString);
94
95 Function *F = M.getFunction("add_x2");
96 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
97 auto Plan = buildHCFG(LoopHeader);
98 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
99
100 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
101 EXPECT_NE(nullptr, Entry->getSingleSuccessor());
102 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
103
104 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
105 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
106
107 VPlanSlp Slp(VPIAI, *Body);
108 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
109 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
110 EXPECT_EQ(64u, Slp.getWidestBundleBits());
111 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
112
113 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
114 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
115
116 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
117 auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1));
118 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
119 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
120
121 delete CombinedStore;
122 delete CombinedAdd;
123 delete CombinedLoadA;
124 delete CombinedLoadB;
125 }
126
TEST_F(VPlanSlpTest,testSlpSimple_3)127 TEST_F(VPlanSlpTest, testSlpSimple_3) {
128 const char *ModuleString =
129 "%struct.Test = type { i32, i32 }\n"
130 "%struct.Test3 = type { i32, i32, i32 }\n"
131 "%struct.Test4xi8 = type { i8, i8, i8 }\n"
132 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
133 "nocapture readonly %B, %struct.Test* nocapture %C) {\n"
134 "entry:\n"
135 " br label %for.body\n"
136 "for.body: ; preds = %for.body, "
137 "%entry\n"
138 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
139 " %A0 = getelementptr %struct.Test, %struct.Test* %A, i64 "
140 " %indvars.iv, i32 0\n"
141 " %vA0 = load i32, i32* %A0, align 4\n"
142 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
143 " %indvars.iv, i32 0\n"
144 " %vB0 = load i32, i32* %B0, align 4\n"
145 " %add0 = add nsw i32 %vA0, %vB0\n"
146 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
147 " %indvars.iv, i32 1\n"
148 " %vA1 = load i32, i32* %A1, align 4\n"
149 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
150 " %indvars.iv, i32 1\n"
151 " %vB1 = load i32, i32* %B1, align 4\n"
152 " %add1 = add nsw i32 %vA1, %vB1\n"
153 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
154 " %indvars.iv, i32 0\n"
155 " store i32 %add0, i32* %C0, align 4\n"
156 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
157 " %indvars.iv, i32 1\n"
158 " store i32 %add1, i32* %C1, align 4\n"
159 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
160 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
161 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
162 "for.cond.cleanup: ; preds = %for.body\n"
163 " ret void\n"
164 "}\n";
165
166 Module &M = parseModule(ModuleString);
167
168 Function *F = M.getFunction("add_x2");
169 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
170 auto Plan = buildHCFG(LoopHeader);
171
172 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
173 EXPECT_NE(nullptr, Entry->getSingleSuccessor());
174 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
175
176 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
177 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
178
179 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
180
181 VPlanSlp Slp(VPIAI, *Body);
182 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
183 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
184 EXPECT_EQ(64u, Slp.getWidestBundleBits());
185 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
186
187 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
188 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
189
190 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
191 auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1));
192 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
193 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
194
195 VPInstruction *GetA = cast<VPInstruction>(&*std::next(Body->begin(), 1));
196 VPInstruction *GetB = cast<VPInstruction>(&*std::next(Body->begin(), 3));
197 EXPECT_EQ(GetA, CombinedLoadA->getOperand(0));
198 EXPECT_EQ(GetB, CombinedLoadB->getOperand(0));
199
200 delete CombinedStore;
201 delete CombinedAdd;
202 delete CombinedLoadA;
203 delete CombinedLoadB;
204 }
205
TEST_F(VPlanSlpTest,testSlpReuse_1)206 TEST_F(VPlanSlpTest, testSlpReuse_1) {
207 const char *ModuleString =
208 "%struct.Test = type { i32, i32 }\n"
209 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
210 "nocapture readonly %B, %struct.Test* nocapture %C) {\n"
211 "entry:\n"
212 " br label %for.body\n"
213 "for.body: ; preds = %for.body, "
214 "%entry\n"
215 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
216 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
217 "%indvars.iv, i32 0\n"
218 " %vA0 = load i32, i32* %A0, align 4\n"
219 " %add0 = add nsw i32 %vA0, %vA0\n"
220 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
221 "%indvars.iv, i32 1\n"
222 " %vA1 = load i32, i32* %A1, align 4\n"
223 " %add1 = add nsw i32 %vA1, %vA1\n"
224 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
225 "%indvars.iv, i32 0\n"
226 " store i32 %add0, i32* %C0, align 4\n"
227 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
228 "%indvars.iv, i32 1\n"
229 " store i32 %add1, i32* %C1, align 4\n"
230 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
231 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
232 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
233 "for.cond.cleanup: ; preds = %for.body\n"
234 " ret void\n"
235 "}\n";
236
237 Module &M = parseModule(ModuleString);
238
239 Function *F = M.getFunction("add_x2");
240 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
241 auto Plan = buildHCFG(LoopHeader);
242 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
243
244 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
245 EXPECT_NE(nullptr, Entry->getSingleSuccessor());
246 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
247
248 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 8));
249 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10));
250
251 VPlanSlp Slp(VPIAI, *Body);
252 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
253 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
254 EXPECT_EQ(64u, Slp.getWidestBundleBits());
255 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
256
257 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
258 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
259
260 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
261 EXPECT_EQ(CombinedLoadA, CombinedAdd->getOperand(1));
262 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
263
264 delete CombinedStore;
265 delete CombinedAdd;
266 delete CombinedLoadA;
267 }
268
TEST_F(VPlanSlpTest,testSlpReuse_2)269 TEST_F(VPlanSlpTest, testSlpReuse_2) {
270 const char *ModuleString =
271 "%struct.Test = type { i32, i32 }\n"
272 "define i32 @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
273 "nocapture readonly %B, %struct.Test* nocapture %C) {\n"
274 "entry:\n"
275 " br label %for.body\n"
276 "for.body: ; preds = %for.body, "
277 "%entry\n"
278 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
279 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
280 "%indvars.iv, i32 0\n"
281 " %vA0 = load i32, i32* %A0, align 4\n"
282 " %add0 = add nsw i32 %vA0, %vA0\n"
283 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
284 "%indvars.iv, i32 0\n"
285 " store i32 %add0, i32* %C0, align 4\n"
286 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
287 "%indvars.iv, i32 1\n"
288 " %vA1 = load i32, i32* %A1, align 4\n"
289 " %add1 = add nsw i32 %vA1, %vA1\n"
290 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
291 "%indvars.iv, i32 1\n"
292 " store i32 %add1, i32* %C1, align 4\n"
293 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
294 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
295 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
296 "for.cond.cleanup: ; preds = %for.body\n"
297 " ret i32 %vA1\n"
298 "}\n";
299
300 Module &M = parseModule(ModuleString);
301
302 Function *F = M.getFunction("add_x2");
303 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
304 auto Plan = buildHCFG(LoopHeader);
305 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
306
307 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
308 EXPECT_NE(nullptr, Entry->getSingleSuccessor());
309 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
310
311 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 5));
312 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10));
313
314 VPlanSlp Slp(VPIAI, *Body);
315 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
316 Slp.buildGraph(StoreRoot);
317 EXPECT_FALSE(Slp.isCompletelySLP());
318 }
319
checkReorderExample(VPInstruction * Store1,VPInstruction * Store2,VPBasicBlock * Body,VPInterleavedAccessInfo && IAI)320 static void checkReorderExample(VPInstruction *Store1, VPInstruction *Store2,
321 VPBasicBlock *Body,
322 VPInterleavedAccessInfo &&IAI) {
323 VPlanSlp Slp(IAI, *Body);
324 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
325 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
326
327 EXPECT_TRUE(Slp.isCompletelySLP());
328 EXPECT_EQ(CombinedStore->getOpcode(), VPInstruction::SLPStore);
329
330 VPInstruction *CombinedAdd =
331 cast<VPInstruction>(CombinedStore->getOperand(0));
332 EXPECT_EQ(CombinedAdd->getOpcode(), Instruction::Add);
333
334 VPInstruction *CombinedMulAB =
335 cast<VPInstruction>(CombinedAdd->getOperand(0));
336 VPInstruction *CombinedMulCD =
337 cast<VPInstruction>(CombinedAdd->getOperand(1));
338 EXPECT_EQ(CombinedMulAB->getOpcode(), Instruction::Mul);
339
340 VPInstruction *CombinedLoadA =
341 cast<VPInstruction>(CombinedMulAB->getOperand(0));
342 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
343 VPInstruction *LoadvA0 = cast<VPInstruction>(&*std::next(Body->begin(), 2));
344 VPInstruction *LoadvA1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
345 EXPECT_EQ(LoadvA0->getOperand(0), CombinedLoadA->getOperand(0));
346 EXPECT_EQ(LoadvA1->getOperand(0), CombinedLoadA->getOperand(1));
347
348 VPInstruction *CombinedLoadB =
349 cast<VPInstruction>(CombinedMulAB->getOperand(1));
350 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
351 VPInstruction *LoadvB0 = cast<VPInstruction>(&*std::next(Body->begin(), 4));
352 VPInstruction *LoadvB1 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
353 EXPECT_EQ(LoadvB0->getOperand(0), CombinedLoadB->getOperand(0));
354 EXPECT_EQ(LoadvB1->getOperand(0), CombinedLoadB->getOperand(1));
355
356 EXPECT_EQ(CombinedMulCD->getOpcode(), Instruction::Mul);
357
358 VPInstruction *CombinedLoadC =
359 cast<VPInstruction>(CombinedMulCD->getOperand(0));
360 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadC->getOpcode());
361 VPInstruction *LoadvC0 = cast<VPInstruction>(&*std::next(Body->begin(), 7));
362 VPInstruction *LoadvC1 = cast<VPInstruction>(&*std::next(Body->begin(), 17));
363 EXPECT_EQ(LoadvC0->getOperand(0), CombinedLoadC->getOperand(0));
364 EXPECT_EQ(LoadvC1->getOperand(0), CombinedLoadC->getOperand(1));
365
366 VPInstruction *CombinedLoadD =
367 cast<VPInstruction>(CombinedMulCD->getOperand(1));
368 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadD->getOpcode());
369 VPInstruction *LoadvD0 = cast<VPInstruction>(&*std::next(Body->begin(), 9));
370 VPInstruction *LoadvD1 = cast<VPInstruction>(&*std::next(Body->begin(), 19));
371 EXPECT_EQ(LoadvD0->getOperand(0), CombinedLoadD->getOperand(0));
372 EXPECT_EQ(LoadvD1->getOperand(0), CombinedLoadD->getOperand(1));
373
374 delete CombinedStore;
375 delete CombinedAdd;
376 delete CombinedMulAB;
377 delete CombinedMulCD;
378 delete CombinedLoadA;
379 delete CombinedLoadB;
380 delete CombinedLoadC;
381 delete CombinedLoadD;
382 }
383
TEST_F(VPlanSlpTest,testSlpReorder_1)384 TEST_F(VPlanSlpTest, testSlpReorder_1) {
385 LLVMContext Ctx;
386 const char *ModuleString =
387 "%struct.Test = type { i32, i32 }\n"
388 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* "
389 "%C, %struct.Test* %D, %struct.Test* %E) {\n"
390 "entry:\n"
391 " br label %for.body\n"
392 "for.body: ; preds = %for.body, "
393 "%entry\n"
394 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
395 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
396 "%indvars.iv, i32 0\n"
397 " %vA0 = load i32, i32* %A0, align 4\n"
398 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
399 "%indvars.iv, i32 0\n"
400 " %vB0 = load i32, i32* %B0, align 4\n"
401 " %mul11 = mul nsw i32 %vA0, %vB0\n"
402 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
403 "%indvars.iv, i32 0\n"
404 " %vC0 = load i32, i32* %C0, align 4\n"
405 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
406 "%indvars.iv, i32 0\n"
407 " %vD0 = load i32, i32* %D0, align 4\n"
408 " %mul12 = mul nsw i32 %vC0, %vD0\n"
409 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
410 "%indvars.iv, i32 1\n"
411 " %vA1 = load i32, i32* %A1, align 4\n"
412 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
413 "%indvars.iv, i32 1\n"
414 " %vB1 = load i32, i32* %B1, align 4\n"
415 " %mul21 = mul nsw i32 %vA1, %vB1\n"
416 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
417 "%indvars.iv, i32 1\n"
418 " %vC1 = load i32, i32* %C1, align 4\n"
419 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
420 "%indvars.iv, i32 1\n"
421 " %vD1 = load i32, i32* %D1, align 4\n"
422 " %mul22 = mul nsw i32 %vC1, %vD1\n"
423 " %add1 = add nsw i32 %mul11, %mul12\n"
424 " %add2 = add nsw i32 %mul22, %mul21\n"
425 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
426 "%indvars.iv, i32 0\n"
427 " store i32 %add1, i32* %E0, align 4\n"
428 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
429 "%indvars.iv, i32 1\n"
430 " store i32 %add2, i32* %E1, align 4\n"
431 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
432 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
433 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
434 "for.cond.cleanup: ; preds = %for.body\n"
435 " ret void\n"
436 "}\n";
437
438 Module &M = parseModule(ModuleString);
439
440 Function *F = M.getFunction("add_x3");
441 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
442 auto Plan = buildHCFG(LoopHeader);
443
444 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
445 EXPECT_NE(nullptr, Entry->getSingleSuccessor());
446 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
447
448 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
449 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
450
451 checkReorderExample(
452 Store1, Store2, Body,
453 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan));
454 }
455
TEST_F(VPlanSlpTest,testSlpReorder_2)456 TEST_F(VPlanSlpTest, testSlpReorder_2) {
457 LLVMContext Ctx;
458 const char *ModuleString =
459 "%struct.Test = type { i32, i32 }\n"
460 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* "
461 "%C, %struct.Test* %D, %struct.Test* %E) {\n"
462 "entry:\n"
463 " br label %for.body\n"
464 "for.body: ; preds = %for.body, "
465 "%entry\n"
466 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
467 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
468 "%indvars.iv, i32 0\n"
469 " %vA0 = load i32, i32* %A0, align 4\n"
470 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
471 "%indvars.iv, i32 0\n"
472 " %vB0 = load i32, i32* %B0, align 4\n"
473 " %mul11 = mul nsw i32 %vA0, %vB0\n"
474 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
475 "%indvars.iv, i32 0\n"
476 " %vC0 = load i32, i32* %C0, align 4\n"
477 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
478 "%indvars.iv, i32 0\n"
479 " %vD0 = load i32, i32* %D0, align 4\n"
480 " %mul12 = mul nsw i32 %vC0, %vD0\n"
481 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
482 "%indvars.iv, i32 1\n"
483 " %vA1 = load i32, i32* %A1, align 4\n"
484 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
485 "%indvars.iv, i32 1\n"
486 " %vB1 = load i32, i32* %B1, align 4\n"
487 " %mul21 = mul nsw i32 %vB1, %vA1\n"
488 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
489 "%indvars.iv, i32 1\n"
490 " %vC1 = load i32, i32* %C1, align 4\n"
491 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
492 "%indvars.iv, i32 1\n"
493 " %vD1 = load i32, i32* %D1, align 4\n"
494 " %mul22 = mul nsw i32 %vD1, %vC1\n"
495 " %add1 = add nsw i32 %mul11, %mul12\n"
496 " %add2 = add nsw i32 %mul22, %mul21\n"
497 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
498 "%indvars.iv, i32 0\n"
499 " store i32 %add1, i32* %E0, align 4\n"
500 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
501 "%indvars.iv, i32 1\n"
502 " store i32 %add2, i32* %E1, align 4\n"
503 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
504 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
505 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
506 "for.cond.cleanup: ; preds = %for.body\n"
507 " ret void\n"
508 "}\n";
509
510 Module &M = parseModule(ModuleString);
511
512 Function *F = M.getFunction("add_x3");
513 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
514 auto Plan = buildHCFG(LoopHeader);
515
516 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
517 EXPECT_NE(nullptr, Entry->getSingleSuccessor());
518 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
519
520 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
521 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
522
523 checkReorderExample(
524 Store1, Store2, Body,
525 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan));
526 }
527
TEST_F(VPlanSlpTest,testSlpReorder_3)528 TEST_F(VPlanSlpTest, testSlpReorder_3) {
529 LLVMContext Ctx;
530 const char *ModuleString =
531 "%struct.Test = type { i32, i32 }\n"
532 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* "
533 "%C, %struct.Test* %D, %struct.Test* %E) {\n"
534 "entry:\n"
535 " br label %for.body\n"
536 "for.body: ; preds = %for.body, "
537 "%entry\n"
538 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
539 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
540 "%indvars.iv, i32 1\n"
541 " %vA1 = load i32, i32* %A1, align 4\n"
542 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
543 "%indvars.iv, i32 0\n"
544 " %vB0 = load i32, i32* %B0, align 4\n"
545 " %mul11 = mul nsw i32 %vA1, %vB0\n"
546 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
547 "%indvars.iv, i32 0\n"
548 " %vC0 = load i32, i32* %C0, align 4\n"
549 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
550 "%indvars.iv, i32 0\n"
551 " %vD0 = load i32, i32* %D0, align 4\n"
552 " %mul12 = mul nsw i32 %vC0, %vD0\n"
553 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
554 "%indvars.iv, i32 0\n"
555 " %vA0 = load i32, i32* %A0, align 4\n"
556 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
557 "%indvars.iv, i32 1\n"
558 " %vB1 = load i32, i32* %B1, align 4\n"
559 " %mul21 = mul nsw i32 %vB1, %vA0\n"
560 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
561 "%indvars.iv, i32 1\n"
562 " %vC1 = load i32, i32* %C1, align 4\n"
563 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
564 "%indvars.iv, i32 1\n"
565 " %vD1 = load i32, i32* %D1, align 4\n"
566 " %mul22 = mul nsw i32 %vD1, %vC1\n"
567 " %add1 = add nsw i32 %mul11, %mul12\n"
568 " %add2 = add nsw i32 %mul22, %mul21\n"
569 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
570 "%indvars.iv, i32 0\n"
571 " store i32 %add1, i32* %E0, align 4\n"
572 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
573 "%indvars.iv, i32 1\n"
574 " store i32 %add2, i32* %E1, align 4\n"
575 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
576 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
577 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
578 "for.cond.cleanup: ; preds = %for.body\n"
579 " ret void\n"
580 "}\n";
581
582 Module &M = parseModule(ModuleString);
583
584 Function *F = M.getFunction("add_x3");
585 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
586 auto Plan = buildHCFG(LoopHeader);
587
588 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
589 EXPECT_NE(nullptr, Entry->getSingleSuccessor());
590 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
591
592 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
593 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
594
595 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
596 VPlanSlp Slp(VPIAI, *Body);
597 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
598 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
599
600 // FIXME Need to select better first value for lane0.
601 EXPECT_FALSE(Slp.isCompletelySLP());
602 }
603
TEST_F(VPlanSlpTest,testSlpReorder_4)604 TEST_F(VPlanSlpTest, testSlpReorder_4) {
605 LLVMContext Ctx;
606 const char *ModuleString =
607 "%struct.Test = type { i32, i32 }\n"
608 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* "
609 "%C, %struct.Test* %D, %struct.Test* %E) {\n"
610 "entry:\n"
611 " br label %for.body\n"
612 "for.body: ; preds = %for.body, "
613 "%entry\n"
614 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
615 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
616 "%indvars.iv, i32 0\n"
617 " %vA0 = load i32, i32* %A0, align 4\n"
618 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
619 "%indvars.iv, i32 0\n"
620 " %vB0 = load i32, i32* %B0, align 4\n"
621 " %mul11 = mul nsw i32 %vA0, %vB0\n"
622 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
623 "%indvars.iv, i32 0\n"
624 " %vC0 = load i32, i32* %C0, align 4\n"
625 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
626 "%indvars.iv, i32 0\n"
627 " %vD0 = load i32, i32* %D0, align 4\n"
628 " %mul12 = mul nsw i32 %vC0, %vD0\n"
629 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
630 "%indvars.iv, i32 1\n"
631 " %vA1 = load i32, i32* %A1, align 4\n"
632 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
633 "%indvars.iv, i32 1\n"
634 " %vB1 = load i32, i32* %B1, align 4\n"
635 " %mul21 = mul nsw i32 %vA1, %vB1\n"
636 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
637 "%indvars.iv, i32 1\n"
638 " %vC1 = load i32, i32* %C1, align 4\n"
639 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
640 "%indvars.iv, i32 1\n"
641 " %vD1 = load i32, i32* %D1, align 4\n"
642 " %mul22 = mul nsw i32 %vC1, %vD1\n"
643 " %add1 = add nsw i32 %mul11, %mul12\n"
644 " %add2 = add nsw i32 %mul22, %mul21\n"
645 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
646 "%indvars.iv, i32 0\n"
647 " store i32 %add1, i32* %E0, align 4\n"
648 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
649 "%indvars.iv, i32 1\n"
650 " store i32 %add2, i32* %E1, align 4\n"
651 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
652 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
653 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
654 "for.cond.cleanup: ; preds = %for.body\n"
655 " ret void\n"
656 "}\n";
657
658 Module &M = parseModule(ModuleString);
659
660 Function *F = M.getFunction("add_x3");
661 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
662 auto Plan = buildHCFG(LoopHeader);
663
664 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
665 EXPECT_NE(nullptr, Entry->getSingleSuccessor());
666 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
667
668 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
669 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
670
671 checkReorderExample(
672 Store1, Store2, Body,
673 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan));
674 }
675
676 // Make sure we do not combine instructions with operands in different BBs.
TEST_F(VPlanSlpTest,testInstrsInDifferentBBs)677 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs) {
678 const char *ModuleString =
679 "%struct.Test = type { i32, i32 }\n"
680 "%struct.Test3 = type { i32, i32, i32 }\n"
681 "%struct.Test4xi8 = type { i8, i8, i8 }\n"
682 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
683 "nocapture readonly %B, %struct.Test* nocapture %C) {\n"
684 "entry:\n"
685 " br label %for.body\n"
686 "for.body: ; preds = %for.body, "
687 "%entry\n"
688 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
689 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
690 "%indvars.iv, i32 0\n"
691 " %vA0 = load i32, i32* %A0, align 4\n"
692 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
693 "%indvars.iv, i32 0\n"
694 " %vB0 = load i32, i32* %B0, align 4\n"
695 " %add0 = add nsw i32 %vA0, %vB0\n"
696 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
697 "%indvars.iv, i32 1\n"
698 " %vA1 = load i32, i32* %A1, align 4\n"
699 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
700 "%indvars.iv, i32 1\n"
701 " br label %bb2\n"
702 "bb2:\n"
703 " %vB1 = load i32, i32* %B1, align 4\n"
704 " %add1 = add nsw i32 %vA1, %vB1\n"
705 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
706 "%indvars.iv, i32 0\n"
707 " store i32 %add0, i32* %C0, align 4\n"
708 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
709 "%indvars.iv, i32 1\n"
710 " store i32 %add1, i32* %C1, align 4\n"
711 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
712 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
713 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
714 "for.cond.cleanup: ; preds = %for.body\n"
715 " ret void\n"
716 "}\n";
717
718 Module &M = parseModule(ModuleString);
719
720 Function *F = M.getFunction("add_x2");
721 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
722 auto Plan = buildHCFG(LoopHeader);
723 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
724
725 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
726 EXPECT_NE(nullptr, Entry->getSingleSuccessor());
727 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
728 VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock();
729
730 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 3));
731 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 5));
732
733 VPlanSlp Slp(VPIAI, *BB2);
734 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
735 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
736 EXPECT_EQ(0u, Slp.getWidestBundleBits());
737 }
738
739 // Make sure we do not combine instructions with operands in different BBs.
TEST_F(VPlanSlpTest,testInstrsInDifferentBBs2)740 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs2) {
741 const char *ModuleString =
742 "%struct.Test = type { i32, i32 }\n"
743 "%struct.Test3 = type { i32, i32, i32 }\n"
744 "%struct.Test4xi8 = type { i8, i8, i8 }\n"
745 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
746 "nocapture readonly %B, %struct.Test* nocapture %C) {\n"
747 "entry:\n"
748 " br label %for.body\n"
749 "for.body: ; preds = %for.body, "
750 "%entry\n"
751 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
752 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
753 "%indvars.iv, i32 0\n"
754 " %vA0 = load i32, i32* %A0, align 4\n"
755 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
756 "%indvars.iv, i32 0\n"
757 " %vB0 = load i32, i32* %B0, align 4\n"
758 " %add0 = add nsw i32 %vA0, %vB0\n"
759 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
760 "%indvars.iv, i32 1\n"
761 " %vA1 = load i32, i32* %A1, align 4\n"
762 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
763 "%indvars.iv, i32 1\n"
764 " %vB1 = load i32, i32* %B1, align 4\n"
765 " %add1 = add nsw i32 %vA1, %vB1\n"
766 " br label %bb2\n"
767 "bb2:\n"
768 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
769 "%indvars.iv, i32 0\n"
770 " store i32 %add0, i32* %C0, align 4\n"
771 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
772 "%indvars.iv, i32 1\n"
773 " store i32 %add1, i32* %C1, align 4\n"
774 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
775 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
776 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
777 "for.cond.cleanup: ; preds = %for.body\n"
778 " ret void\n"
779 "}\n";
780
781 Module &M = parseModule(ModuleString);
782
783 Function *F = M.getFunction("add_x2");
784 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
785 auto Plan = buildHCFG(LoopHeader);
786 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
787
788 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
789 EXPECT_NE(nullptr, Entry->getSingleSuccessor());
790 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
791 VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock();
792
793 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 1));
794 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 3));
795
796 VPlanSlp Slp(VPIAI, *BB2);
797 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
798 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
799 EXPECT_EQ(0u, Slp.getWidestBundleBits());
800 }
801
TEST_F(VPlanSlpTest,testSlpAtomicLoad)802 TEST_F(VPlanSlpTest, testSlpAtomicLoad) {
803 const char *ModuleString =
804 "%struct.Test = type { i32, i32 }\n"
805 "%struct.Test3 = type { i32, i32, i32 }\n"
806 "%struct.Test4xi8 = type { i8, i8, i8 }\n"
807 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
808 "nocapture readonly %B, %struct.Test* nocapture %C) {\n"
809 "entry:\n"
810 " br label %for.body\n"
811 "for.body: ; preds = %for.body, "
812 "%entry\n"
813 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
814 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
815 "%indvars.iv, i32 0\n"
816 " %vA0 = load atomic i32, i32* %A0 monotonic, align 4\n"
817 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
818 "%indvars.iv, i32 0\n"
819 " %vB0 = load i32, i32* %B0, align 4\n"
820 " %add0 = add nsw i32 %vA0, %vB0\n"
821 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
822 "%indvars.iv, i32 1\n"
823 " %vA1 = load i32, i32* %A1, align 4\n"
824 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
825 "%indvars.iv, i32 1\n"
826 " %vB1 = load i32, i32* %B1, align 4\n"
827 " %add1 = add nsw i32 %vA1, %vB1\n"
828 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
829 "%indvars.iv, i32 0\n"
830 " store i32 %add0, i32* %C0, align 4\n"
831 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
832 "%indvars.iv, i32 1\n"
833 " store i32 %add1, i32* %C1, align 4\n"
834 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
835 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
836 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
837 "for.cond.cleanup: ; preds = %for.body\n"
838 " ret void\n"
839 "}\n";
840
841 Module &M = parseModule(ModuleString);
842
843 Function *F = M.getFunction("add_x2");
844 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
845 auto Plan = buildHCFG(LoopHeader);
846 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
847
848 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
849 EXPECT_NE(nullptr, Entry->getSingleSuccessor());
850 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
851
852 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
853 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
854
855 VPlanSlp Slp(VPIAI, *Body);
856 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
857 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
858 EXPECT_FALSE(Slp.isCompletelySLP());
859 }
860
TEST_F(VPlanSlpTest,testSlpAtomicStore)861 TEST_F(VPlanSlpTest, testSlpAtomicStore) {
862 const char *ModuleString =
863 "%struct.Test = type { i32, i32 }\n"
864 "%struct.Test3 = type { i32, i32, i32 }\n"
865 "%struct.Test4xi8 = type { i8, i8, i8 }\n"
866 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
867 "nocapture readonly %B, %struct.Test* nocapture %C) {\n"
868 "entry:\n"
869 " br label %for.body\n"
870 "for.body: ; preds = %for.body, "
871 "%entry\n"
872 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
873 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
874 "%indvars.iv, i32 0\n"
875 " %vA0 = load i32, i32* %A0, align 4\n"
876 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
877 "%indvars.iv, i32 0\n"
878 " %vB0 = load i32, i32* %B0, align 4\n"
879 " %add0 = add nsw i32 %vA0, %vB0\n"
880 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
881 "%indvars.iv, i32 1\n"
882 " %vA1 = load i32, i32* %A1, align 4\n"
883 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
884 "%indvars.iv, i32 1\n"
885 " %vB1 = load i32, i32* %B1, align 4\n"
886 " %add1 = add nsw i32 %vA1, %vB1\n"
887 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
888 "%indvars.iv, i32 0\n"
889 " store atomic i32 %add0, i32* %C0 monotonic, align 4\n"
890 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
891 "%indvars.iv, i32 1\n"
892 " store i32 %add1, i32* %C1, align 4\n"
893 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
894 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
895 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
896 "for.cond.cleanup: ; preds = %for.body\n"
897 " ret void\n"
898 "}\n";
899
900 Module &M = parseModule(ModuleString);
901
902 Function *F = M.getFunction("add_x2");
903 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
904 auto Plan = buildHCFG(LoopHeader);
905 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
906
907 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
908 EXPECT_NE(nullptr, Entry->getSingleSuccessor());
909 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
910
911 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
912 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
913
914 VPlanSlp Slp(VPIAI, *Body);
915 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
916 Slp.buildGraph(StoreRoot);
917 EXPECT_FALSE(Slp.isCompletelySLP());
918 }
919
920 } // namespace
921 } // namespace llvm
922