1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "CGLoopInfo.h"
11 #include "clang/AST/ASTContext.h"
12 #include "clang/AST/Attr.h"
13 #include "clang/Sema/LoopHint.h"
14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/Constants.h"
16 #include "llvm/IR/InstrTypes.h"
17 #include "llvm/IR/Instructions.h"
18 #include "llvm/IR/Metadata.h"
19 using namespace clang::CodeGen;
20 using namespace llvm;
21 
createMetadata(LLVMContext & Ctx,const LoopAttributes & Attrs)22 static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) {
23 
24   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
25       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
26       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
27       Attrs.UnrollEnable == LoopAttributes::Unspecified)
28     return nullptr;
29 
30   SmallVector<Metadata *, 4> Args;
31   // Reserve operand 0 for loop id self reference.
32   auto TempNode = MDNode::getTemporary(Ctx, None);
33   Args.push_back(TempNode.get());
34 
35   // Setting vectorize.width
36   if (Attrs.VectorizeWidth > 0) {
37     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
38                         ConstantAsMetadata::get(ConstantInt::get(
39                             Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
40     Args.push_back(MDNode::get(Ctx, Vals));
41   }
42 
43   // Setting interleave.count
44   if (Attrs.InterleaveCount > 0) {
45     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
46                         ConstantAsMetadata::get(ConstantInt::get(
47                             Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
48     Args.push_back(MDNode::get(Ctx, Vals));
49   }
50 
51   // Setting interleave.count
52   if (Attrs.UnrollCount > 0) {
53     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
54                         ConstantAsMetadata::get(ConstantInt::get(
55                             Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
56     Args.push_back(MDNode::get(Ctx, Vals));
57   }
58 
59   // Setting vectorize.enable
60   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
61     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
62                         ConstantAsMetadata::get(ConstantInt::get(
63                             Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
64                                                    LoopAttributes::Enable)))};
65     Args.push_back(MDNode::get(Ctx, Vals));
66   }
67 
68   // Setting unroll.full or unroll.disable
69   if (Attrs.UnrollEnable != LoopAttributes::Unspecified) {
70     std::string Name;
71     if (Attrs.UnrollEnable == LoopAttributes::Enable)
72       Name = "llvm.loop.unroll.enable";
73     else if (Attrs.UnrollEnable == LoopAttributes::Full)
74       Name = "llvm.loop.unroll.full";
75     else
76       Name = "llvm.loop.unroll.disable";
77     Metadata *Vals[] = {MDString::get(Ctx, Name)};
78     Args.push_back(MDNode::get(Ctx, Vals));
79   }
80 
81   // Set the first operand to itself.
82   MDNode *LoopID = MDNode::get(Ctx, Args);
83   LoopID->replaceOperandWith(0, LoopID);
84   return LoopID;
85 }
86 
LoopAttributes(bool IsParallel)87 LoopAttributes::LoopAttributes(bool IsParallel)
88     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
89       UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
90       InterleaveCount(0), UnrollCount(0) {}
91 
clear()92 void LoopAttributes::clear() {
93   IsParallel = false;
94   VectorizeWidth = 0;
95   InterleaveCount = 0;
96   UnrollCount = 0;
97   VectorizeEnable = LoopAttributes::Unspecified;
98   UnrollEnable = LoopAttributes::Unspecified;
99 }
100 
LoopInfo(BasicBlock * Header,const LoopAttributes & Attrs)101 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs)
102     : LoopID(nullptr), Header(Header), Attrs(Attrs) {
103   LoopID = createMetadata(Header->getContext(), Attrs);
104 }
105 
push(BasicBlock * Header)106 void LoopInfoStack::push(BasicBlock *Header) {
107   Active.push_back(LoopInfo(Header, StagedAttrs));
108   // Clear the attributes so nested loops do not inherit them.
109   StagedAttrs.clear();
110 }
111 
push(BasicBlock * Header,clang::ASTContext & Ctx,ArrayRef<const clang::Attr * > Attrs)112 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
113                          ArrayRef<const clang::Attr *> Attrs) {
114 
115   // Identify loop hint attributes from Attrs.
116   for (const auto *Attr : Attrs) {
117     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
118 
119     // Skip non loop hint attributes
120     if (!LH)
121       continue;
122 
123     auto *ValueExpr = LH->getValue();
124     unsigned ValueInt = 1;
125     if (ValueExpr) {
126       llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
127       ValueInt = ValueAPS.getSExtValue();
128     }
129 
130     LoopHintAttr::OptionType Option = LH->getOption();
131     LoopHintAttr::LoopHintState State = LH->getState();
132     switch (State) {
133     case LoopHintAttr::Disable:
134       switch (Option) {
135       case LoopHintAttr::Vectorize:
136         // Disable vectorization by specifying a width of 1.
137         setVectorizeWidth(1);
138         break;
139       case LoopHintAttr::Interleave:
140         // Disable interleaving by speciyfing a count of 1.
141         setInterleaveCount(1);
142         break;
143       case LoopHintAttr::Unroll:
144         setUnrollState(LoopAttributes::Disable);
145         break;
146       case LoopHintAttr::UnrollCount:
147       case LoopHintAttr::VectorizeWidth:
148       case LoopHintAttr::InterleaveCount:
149         llvm_unreachable("Options cannot be disabled.");
150         break;
151       }
152       break;
153     case LoopHintAttr::Enable:
154       switch (Option) {
155       case LoopHintAttr::Vectorize:
156       case LoopHintAttr::Interleave:
157         setVectorizeEnable(true);
158         break;
159       case LoopHintAttr::Unroll:
160         setUnrollState(LoopAttributes::Enable);
161         break;
162       case LoopHintAttr::UnrollCount:
163       case LoopHintAttr::VectorizeWidth:
164       case LoopHintAttr::InterleaveCount:
165         llvm_unreachable("Options cannot enabled.");
166         break;
167       }
168       break;
169     case LoopHintAttr::AssumeSafety:
170       switch (Option) {
171       case LoopHintAttr::Vectorize:
172       case LoopHintAttr::Interleave:
173         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
174         setParallel(true);
175         setVectorizeEnable(true);
176         break;
177       case LoopHintAttr::Unroll:
178       case LoopHintAttr::UnrollCount:
179       case LoopHintAttr::VectorizeWidth:
180       case LoopHintAttr::InterleaveCount:
181         llvm_unreachable("Options cannot be used to assume mem safety.");
182         break;
183       }
184       break;
185     case LoopHintAttr::Full:
186       switch (Option) {
187       case LoopHintAttr::Unroll:
188         setUnrollState(LoopAttributes::Full);
189         break;
190       case LoopHintAttr::Vectorize:
191       case LoopHintAttr::Interleave:
192       case LoopHintAttr::UnrollCount:
193       case LoopHintAttr::VectorizeWidth:
194       case LoopHintAttr::InterleaveCount:
195         llvm_unreachable("Options cannot be used with 'full' hint.");
196         break;
197       }
198       break;
199     case LoopHintAttr::Numeric:
200       switch (Option) {
201       case LoopHintAttr::VectorizeWidth:
202         setVectorizeWidth(ValueInt);
203         break;
204       case LoopHintAttr::InterleaveCount:
205         setInterleaveCount(ValueInt);
206         break;
207       case LoopHintAttr::UnrollCount:
208         setUnrollCount(ValueInt);
209         break;
210       case LoopHintAttr::Unroll:
211       case LoopHintAttr::Vectorize:
212       case LoopHintAttr::Interleave:
213         llvm_unreachable("Options cannot be assigned a value.");
214         break;
215       }
216       break;
217     }
218   }
219 
220   /// Stage the attributes.
221   push(Header);
222 }
223 
pop()224 void LoopInfoStack::pop() {
225   assert(!Active.empty() && "No active loops to pop");
226   Active.pop_back();
227 }
228 
InsertHelper(Instruction * I) const229 void LoopInfoStack::InsertHelper(Instruction *I) const {
230   if (!hasInfo())
231     return;
232 
233   const LoopInfo &L = getInfo();
234   if (!L.getLoopID())
235     return;
236 
237   if (TerminatorInst *TI = dyn_cast<TerminatorInst>(I)) {
238     for (unsigned i = 0, ie = TI->getNumSuccessors(); i < ie; ++i)
239       if (TI->getSuccessor(i) == L.getHeader()) {
240         TI->setMetadata("llvm.loop", L.getLoopID());
241         break;
242       }
243     return;
244   }
245 
246   if (L.getAttributes().IsParallel && I->mayReadOrWriteMemory())
247     I->setMetadata("llvm.mem.parallel_loop_access", L.getLoopID());
248 }
249