1 //===-- SIFormMemoryClauses.cpp -------------------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// This pass creates bundles of SMEM and VMEM instructions forming memory
12 /// clauses if XNACK is enabled. Def operands of clauses are marked as early
13 /// clobber to make sure we will not override any source within a clause.
14 ///
15 //===----------------------------------------------------------------------===//
16
17 #include "AMDGPU.h"
18 #include "AMDGPUSubtarget.h"
19 #include "GCNRegPressure.h"
20 #include "SIInstrInfo.h"
21 #include "SIMachineFunctionInfo.h"
22 #include "SIRegisterInfo.h"
23 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
24 #include "llvm/ADT/DenseMap.h"
25 #include "llvm/CodeGen/LiveIntervals.h"
26 #include "llvm/CodeGen/MachineFunctionPass.h"
27
28 using namespace llvm;
29
30 #define DEBUG_TYPE "si-form-memory-clauses"
31
32 // Clauses longer then 15 instructions would overflow one of the counters
33 // and stall. They can stall even earlier if there are outstanding counters.
34 static cl::opt<unsigned>
35 MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(15),
36 cl::desc("Maximum length of a memory clause, instructions"));
37
38 namespace {
39
40 class SIFormMemoryClauses : public MachineFunctionPass {
41 typedef DenseMap<unsigned, std::pair<unsigned, LaneBitmask>> RegUse;
42
43 public:
44 static char ID;
45
46 public:
SIFormMemoryClauses()47 SIFormMemoryClauses() : MachineFunctionPass(ID) {
48 initializeSIFormMemoryClausesPass(*PassRegistry::getPassRegistry());
49 }
50
51 bool runOnMachineFunction(MachineFunction &MF) override;
52
getPassName() const53 StringRef getPassName() const override {
54 return "SI Form memory clauses";
55 }
56
getAnalysisUsage(AnalysisUsage & AU) const57 void getAnalysisUsage(AnalysisUsage &AU) const override {
58 AU.addRequired<LiveIntervals>();
59 AU.setPreservesAll();
60 MachineFunctionPass::getAnalysisUsage(AU);
61 }
62
63 private:
64 template <typename Callable>
65 void forAllLanes(unsigned Reg, LaneBitmask LaneMask, Callable Func) const;
66
67 bool canBundle(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
68 bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
69 void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
70 bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
71 GCNDownwardRPTracker &RPT);
72
73 const GCNSubtarget *ST;
74 const SIRegisterInfo *TRI;
75 const MachineRegisterInfo *MRI;
76 SIMachineFunctionInfo *MFI;
77
78 unsigned LastRecordedOccupancy;
79 unsigned MaxVGPRs;
80 unsigned MaxSGPRs;
81 };
82
83 } // End anonymous namespace.
84
85 INITIALIZE_PASS_BEGIN(SIFormMemoryClauses, DEBUG_TYPE,
86 "SI Form memory clauses", false, false)
87 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
88 INITIALIZE_PASS_END(SIFormMemoryClauses, DEBUG_TYPE,
89 "SI Form memory clauses", false, false)
90
91
92 char SIFormMemoryClauses::ID = 0;
93
94 char &llvm::SIFormMemoryClausesID = SIFormMemoryClauses::ID;
95
createSIFormMemoryClausesPass()96 FunctionPass *llvm::createSIFormMemoryClausesPass() {
97 return new SIFormMemoryClauses();
98 }
99
isVMEMClauseInst(const MachineInstr & MI)100 static bool isVMEMClauseInst(const MachineInstr &MI) {
101 return SIInstrInfo::isFLAT(MI) || SIInstrInfo::isVMEM(MI);
102 }
103
isSMEMClauseInst(const MachineInstr & MI)104 static bool isSMEMClauseInst(const MachineInstr &MI) {
105 return SIInstrInfo::isSMRD(MI);
106 }
107
108 // There no sense to create store clauses, they do not define anything,
109 // thus there is nothing to set early-clobber.
isValidClauseInst(const MachineInstr & MI,bool IsVMEMClause)110 static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
111 if (MI.isDebugValue() || MI.isBundled())
112 return false;
113 if (!MI.mayLoad() || MI.mayStore())
114 return false;
115 if (AMDGPU::getAtomicNoRetOp(MI.getOpcode()) != -1 ||
116 AMDGPU::getAtomicRetOp(MI.getOpcode()) != -1)
117 return false;
118 if (IsVMEMClause && !isVMEMClauseInst(MI))
119 return false;
120 if (!IsVMEMClause && !isSMEMClauseInst(MI))
121 return false;
122 return true;
123 }
124
getMopState(const MachineOperand & MO)125 static unsigned getMopState(const MachineOperand &MO) {
126 unsigned S = 0;
127 if (MO.isImplicit())
128 S |= RegState::Implicit;
129 if (MO.isDead())
130 S |= RegState::Dead;
131 if (MO.isUndef())
132 S |= RegState::Undef;
133 if (MO.isKill())
134 S |= RegState::Kill;
135 if (MO.isEarlyClobber())
136 S |= RegState::EarlyClobber;
137 if (TargetRegisterInfo::isPhysicalRegister(MO.getReg()) && MO.isRenamable())
138 S |= RegState::Renamable;
139 return S;
140 }
141
142 template <typename Callable>
forAllLanes(unsigned Reg,LaneBitmask LaneMask,Callable Func) const143 void SIFormMemoryClauses::forAllLanes(unsigned Reg, LaneBitmask LaneMask,
144 Callable Func) const {
145 if (LaneMask.all() || TargetRegisterInfo::isPhysicalRegister(Reg) ||
146 LaneMask == MRI->getMaxLaneMaskForVReg(Reg)) {
147 Func(0);
148 return;
149 }
150
151 const TargetRegisterClass *RC = MRI->getRegClass(Reg);
152 unsigned E = TRI->getNumSubRegIndices();
153 SmallVector<unsigned, AMDGPU::NUM_TARGET_SUBREGS> CoveringSubregs;
154 for (unsigned Idx = 1; Idx < E; ++Idx) {
155 // Is this index even compatible with the given class?
156 if (TRI->getSubClassWithSubReg(RC, Idx) != RC)
157 continue;
158 LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
159 // Early exit if we found a perfect match.
160 if (SubRegMask == LaneMask) {
161 Func(Idx);
162 return;
163 }
164
165 if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none())
166 continue;
167
168 CoveringSubregs.push_back(Idx);
169 }
170
171 llvm::sort(CoveringSubregs.begin(), CoveringSubregs.end(),
172 [this](unsigned A, unsigned B) {
173 LaneBitmask MaskA = TRI->getSubRegIndexLaneMask(A);
174 LaneBitmask MaskB = TRI->getSubRegIndexLaneMask(B);
175 unsigned NA = MaskA.getNumLanes();
176 unsigned NB = MaskB.getNumLanes();
177 if (NA != NB)
178 return NA > NB;
179 return MaskA.getHighestLane() > MaskB.getHighestLane();
180 });
181
182 for (unsigned Idx : CoveringSubregs) {
183 LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
184 if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none())
185 continue;
186
187 Func(Idx);
188 LaneMask &= ~SubRegMask;
189 if (LaneMask.none())
190 return;
191 }
192
193 llvm_unreachable("Failed to find all subregs to cover lane mask");
194 }
195
196 // Returns false if there is a use of a def already in the map.
197 // In this case we must break the clause.
canBundle(const MachineInstr & MI,RegUse & Defs,RegUse & Uses) const198 bool SIFormMemoryClauses::canBundle(const MachineInstr &MI,
199 RegUse &Defs, RegUse &Uses) const {
200 // Check interference with defs.
201 for (const MachineOperand &MO : MI.operands()) {
202 // TODO: Prologue/Epilogue Insertion pass does not process bundled
203 // instructions.
204 if (MO.isFI())
205 return false;
206
207 if (!MO.isReg())
208 continue;
209
210 unsigned Reg = MO.getReg();
211
212 // If it is tied we will need to write same register as we read.
213 if (MO.isTied())
214 return false;
215
216 RegUse &Map = MO.isDef() ? Uses : Defs;
217 auto Conflict = Map.find(Reg);
218 if (Conflict == Map.end())
219 continue;
220
221 if (TargetRegisterInfo::isPhysicalRegister(Reg))
222 return false;
223
224 LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
225 if ((Conflict->second.second & Mask).any())
226 return false;
227 }
228
229 return true;
230 }
231
232 // Since all defs in the clause are early clobber we can run out of registers.
233 // Function returns false if pressure would hit the limit if instruction is
234 // bundled into a memory clause.
checkPressure(const MachineInstr & MI,GCNDownwardRPTracker & RPT)235 bool SIFormMemoryClauses::checkPressure(const MachineInstr &MI,
236 GCNDownwardRPTracker &RPT) {
237 // NB: skip advanceBeforeNext() call. Since all defs will be marked
238 // early-clobber they will all stay alive at least to the end of the
239 // clause. Therefor we should not decrease pressure even if load
240 // pointer becomes dead and could otherwise be reused for destination.
241 RPT.advanceToNext();
242 GCNRegPressure MaxPressure = RPT.moveMaxPressure();
243 unsigned Occupancy = MaxPressure.getOccupancy(*ST);
244 if (Occupancy >= MFI->getMinAllowedOccupancy() &&
245 MaxPressure.getVGPRNum() <= MaxVGPRs &&
246 MaxPressure.getSGPRNum() <= MaxSGPRs) {
247 LastRecordedOccupancy = Occupancy;
248 return true;
249 }
250 return false;
251 }
252
253 // Collect register defs and uses along with their lane masks and states.
collectRegUses(const MachineInstr & MI,RegUse & Defs,RegUse & Uses) const254 void SIFormMemoryClauses::collectRegUses(const MachineInstr &MI,
255 RegUse &Defs, RegUse &Uses) const {
256 for (const MachineOperand &MO : MI.operands()) {
257 if (!MO.isReg())
258 continue;
259 unsigned Reg = MO.getReg();
260 if (!Reg)
261 continue;
262
263 LaneBitmask Mask = TargetRegisterInfo::isVirtualRegister(Reg) ?
264 TRI->getSubRegIndexLaneMask(MO.getSubReg()) :
265 LaneBitmask::getAll();
266 RegUse &Map = MO.isDef() ? Defs : Uses;
267
268 auto Loc = Map.find(Reg);
269 unsigned State = getMopState(MO);
270 if (Loc == Map.end()) {
271 Map[Reg] = std::make_pair(State, Mask);
272 } else {
273 Loc->second.first |= State;
274 Loc->second.second |= Mask;
275 }
276 }
277 }
278
279 // Check register def/use conflicts, occupancy limits and collect def/use maps.
280 // Return true if instruction can be bundled with previous. It it cannot
281 // def/use maps are not updated.
processRegUses(const MachineInstr & MI,RegUse & Defs,RegUse & Uses,GCNDownwardRPTracker & RPT)282 bool SIFormMemoryClauses::processRegUses(const MachineInstr &MI,
283 RegUse &Defs, RegUse &Uses,
284 GCNDownwardRPTracker &RPT) {
285 if (!canBundle(MI, Defs, Uses))
286 return false;
287
288 if (!checkPressure(MI, RPT))
289 return false;
290
291 collectRegUses(MI, Defs, Uses);
292 return true;
293 }
294
runOnMachineFunction(MachineFunction & MF)295 bool SIFormMemoryClauses::runOnMachineFunction(MachineFunction &MF) {
296 if (skipFunction(MF.getFunction()))
297 return false;
298
299 ST = &MF.getSubtarget<GCNSubtarget>();
300 if (!ST->isXNACKEnabled())
301 return false;
302
303 const SIInstrInfo *TII = ST->getInstrInfo();
304 TRI = ST->getRegisterInfo();
305 MRI = &MF.getRegInfo();
306 MFI = MF.getInfo<SIMachineFunctionInfo>();
307 LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
308 SlotIndexes *Ind = LIS->getSlotIndexes();
309 bool Changed = false;
310
311 MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count();
312 MaxSGPRs = TRI->getAllocatableSet(MF, &AMDGPU::SGPR_32RegClass).count();
313
314 for (MachineBasicBlock &MBB : MF) {
315 MachineBasicBlock::instr_iterator Next;
316 for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
317 MachineInstr &MI = *I;
318 Next = std::next(I);
319
320 bool IsVMEM = isVMEMClauseInst(MI);
321
322 if (!isValidClauseInst(MI, IsVMEM))
323 continue;
324
325 RegUse Defs, Uses;
326 GCNDownwardRPTracker RPT(*LIS);
327 RPT.reset(MI);
328
329 if (!processRegUses(MI, Defs, Uses, RPT))
330 continue;
331
332 unsigned Length = 1;
333 for ( ; Next != E && Length < MaxClause; ++Next) {
334 if (!isValidClauseInst(*Next, IsVMEM))
335 break;
336
337 // A load from pointer which was loaded inside the same bundle is an
338 // impossible clause because we will need to write and read the same
339 // register inside. In this case processRegUses will return false.
340 if (!processRegUses(*Next, Defs, Uses, RPT))
341 break;
342
343 ++Length;
344 }
345 if (Length < 2)
346 continue;
347
348 Changed = true;
349 MFI->limitOccupancy(LastRecordedOccupancy);
350
351 auto B = BuildMI(MBB, I, DebugLoc(), TII->get(TargetOpcode::BUNDLE));
352 Ind->insertMachineInstrInMaps(*B);
353
354 for (auto BI = I; BI != Next; ++BI) {
355 BI->bundleWithPred();
356 Ind->removeSingleMachineInstrFromMaps(*BI);
357
358 for (MachineOperand &MO : BI->defs())
359 if (MO.readsReg())
360 MO.setIsInternalRead(true);
361 }
362
363 for (auto &&R : Defs) {
364 forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
365 unsigned S = R.second.first | RegState::EarlyClobber;
366 if (!SubReg)
367 S &= ~(RegState::Undef | RegState::Dead);
368 B.addDef(R.first, S, SubReg);
369 });
370 }
371
372 for (auto &&R : Uses) {
373 forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
374 B.addUse(R.first, R.second.first & ~RegState::Kill, SubReg);
375 });
376 }
377
378 for (auto &&R : Defs) {
379 unsigned Reg = R.first;
380 Uses.erase(Reg);
381 if (TargetRegisterInfo::isPhysicalRegister(Reg))
382 continue;
383 LIS->removeInterval(Reg);
384 LIS->createAndComputeVirtRegInterval(Reg);
385 }
386
387 for (auto &&R : Uses) {
388 unsigned Reg = R.first;
389 if (TargetRegisterInfo::isPhysicalRegister(Reg))
390 continue;
391 LIS->removeInterval(Reg);
392 LIS->createAndComputeVirtRegInterval(Reg);
393 }
394 }
395 }
396
397 return Changed;
398 }
399