1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //==-----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// \brief Defines an instruction selector for the AMDGPU target.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "AMDGPUInstrInfo.h"
16 #include "AMDGPUIntrinsicInfo.h"
17 #include "AMDGPUISelLowering.h" // For AMDGPUISD
18 #include "AMDGPUSubtarget.h"
19 #include "SIISelLowering.h"
20 #include "SIMachineFunctionInfo.h"
21 #include "llvm/Analysis/ValueTracking.h"
22 #include "llvm/CodeGen/FunctionLoweringInfo.h"
23 #include "llvm/CodeGen/MachineFrameInfo.h"
24 #include "llvm/CodeGen/PseudoSourceValue.h"
25 #include "llvm/CodeGen/SelectionDAG.h"
26 #include "llvm/CodeGen/SelectionDAGISel.h"
27 #include "llvm/IR/DiagnosticInfo.h"
28 
29 using namespace llvm;
30 
31 namespace llvm {
32 class R600InstrInfo;
33 }
34 
35 //===----------------------------------------------------------------------===//
36 // Instruction Selector Implementation
37 //===----------------------------------------------------------------------===//
38 
39 namespace {
40 
isCBranchSCC(const SDNode * N)41 static bool isCBranchSCC(const SDNode *N) {
42   assert(N->getOpcode() == ISD::BRCOND);
43   if (!N->hasOneUse())
44     return false;
45 
46   SDValue Cond = N->getOperand(1);
47   if (Cond.getOpcode() == ISD::CopyToReg)
48     Cond = Cond.getOperand(2);
49   return Cond.getOpcode() == ISD::SETCC &&
50          Cond.getOperand(0).getValueType() == MVT::i32 && Cond.hasOneUse();
51 }
52 
53 /// AMDGPU specific code to select AMDGPU machine instructions for
54 /// SelectionDAG operations.
55 class AMDGPUDAGToDAGISel : public SelectionDAGISel {
56   // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
57   // make the right decision when generating code for different targets.
58   const AMDGPUSubtarget *Subtarget;
59 
60 public:
61   AMDGPUDAGToDAGISel(TargetMachine &TM);
62   virtual ~AMDGPUDAGToDAGISel();
63   bool runOnMachineFunction(MachineFunction &MF) override;
64   void Select(SDNode *N) override;
65   const char *getPassName() const override;
66   void PreprocessISelDAG() override;
67   void PostprocessISelDAG() override;
68 
69 private:
70   bool isInlineImmediate(SDNode *N) const;
71   bool FoldOperand(SDValue &Src, SDValue &Sel, SDValue &Neg, SDValue &Abs,
72                    const R600InstrInfo *TII);
73   bool FoldOperands(unsigned, const R600InstrInfo *, std::vector<SDValue> &);
74   bool FoldDotOperands(unsigned, const R600InstrInfo *, std::vector<SDValue> &);
75 
76   bool isConstantLoad(const MemSDNode *N, int cbID) const;
77   bool isUniformBr(const SDNode *N) const;
78 
79   SDNode *glueCopyToM0(SDNode *N) const;
80 
81   const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
82   bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
83   bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
84                                        SDValue& Offset);
85   bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
86   bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
87   bool isDSOffsetLegal(const SDValue &Base, unsigned Offset,
88                        unsigned OffsetBits) const;
89   bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
90   bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
91                                  SDValue &Offset1) const;
92   bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
93                    SDValue &SOffset, SDValue &Offset, SDValue &Offen,
94                    SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
95                    SDValue &TFE) const;
96   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
97                          SDValue &SOffset, SDValue &Offset, SDValue &GLC,
98                          SDValue &SLC, SDValue &TFE) const;
99   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
100                          SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
101                          SDValue &SLC) const;
102   bool SelectMUBUFScratch(SDValue Addr, SDValue &RSrc, SDValue &VAddr,
103                           SDValue &SOffset, SDValue &ImmOffset) const;
104   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
105                          SDValue &Offset, SDValue &GLC, SDValue &SLC,
106                          SDValue &TFE) const;
107   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
108                          SDValue &Offset, SDValue &SLC) const;
109   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
110                          SDValue &Offset) const;
111   bool SelectMUBUFConstant(SDValue Constant,
112                            SDValue &SOffset,
113                            SDValue &ImmOffset) const;
114   bool SelectMUBUFIntrinsicOffset(SDValue Offset, SDValue &SOffset,
115                                   SDValue &ImmOffset) const;
116   bool SelectMUBUFIntrinsicVOffset(SDValue Offset, SDValue &SOffset,
117                                    SDValue &ImmOffset, SDValue &VOffset) const;
118 
119   bool SelectFlat(SDValue Addr, SDValue &VAddr,
120                   SDValue &SLC, SDValue &TFE) const;
121 
122   bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
123                         bool &Imm) const;
124   bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
125                   bool &Imm) const;
126   bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
127   bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
128   bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
129   bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
130   bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
131   bool SelectSMRDBufferSgpr(SDValue Addr, SDValue &Offset) const;
132   bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
133   bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
134   bool SelectVOP3NoMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
135   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
136                        SDValue &Clamp, SDValue &Omod) const;
137   bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
138                          SDValue &Clamp, SDValue &Omod) const;
139 
140   bool SelectVOP3Mods0Clamp(SDValue In, SDValue &Src, SDValue &SrcMods,
141                             SDValue &Omod) const;
142   bool SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, SDValue &SrcMods,
143                                  SDValue &Clamp,
144                                  SDValue &Omod) const;
145 
146   void SelectADD_SUB_I64(SDNode *N);
147   void SelectDIV_SCALE(SDNode *N);
148 
149   SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val,
150                    uint32_t Offset, uint32_t Width);
151   void SelectS_BFEFromShifts(SDNode *N);
152   void SelectS_BFE(SDNode *N);
153   void SelectBRCOND(SDNode *N);
154   void SelectATOMIC_CMP_SWAP(SDNode *N);
155 
156   // Include the pieces autogenerated from the target description.
157 #include "AMDGPUGenDAGISel.inc"
158 };
159 }  // end anonymous namespace
160 
161 /// \brief This pass converts a legalized DAG into a AMDGPU-specific
162 // DAG, ready for instruction scheduling.
createAMDGPUISelDag(TargetMachine & TM)163 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine &TM) {
164   return new AMDGPUDAGToDAGISel(TM);
165 }
166 
AMDGPUDAGToDAGISel(TargetMachine & TM)167 AMDGPUDAGToDAGISel::AMDGPUDAGToDAGISel(TargetMachine &TM)
168     : SelectionDAGISel(TM) {}
169 
runOnMachineFunction(MachineFunction & MF)170 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
171   Subtarget = &MF.getSubtarget<AMDGPUSubtarget>();
172   return SelectionDAGISel::runOnMachineFunction(MF);
173 }
174 
~AMDGPUDAGToDAGISel()175 AMDGPUDAGToDAGISel::~AMDGPUDAGToDAGISel() {
176 }
177 
isInlineImmediate(SDNode * N) const178 bool AMDGPUDAGToDAGISel::isInlineImmediate(SDNode *N) const {
179   const SITargetLowering *TL
180       = static_cast<const SITargetLowering *>(getTargetLowering());
181   return TL->analyzeImmediate(N) == 0;
182 }
183 
184 /// \brief Determine the register class for \p OpNo
185 /// \returns The register class of the virtual register that will be used for
186 /// the given operand number \OpNo or NULL if the register class cannot be
187 /// determined.
getOperandRegClass(SDNode * N,unsigned OpNo) const188 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
189                                                           unsigned OpNo) const {
190   if (!N->isMachineOpcode())
191     return nullptr;
192 
193   switch (N->getMachineOpcode()) {
194   default: {
195     const MCInstrDesc &Desc =
196         Subtarget->getInstrInfo()->get(N->getMachineOpcode());
197     unsigned OpIdx = Desc.getNumDefs() + OpNo;
198     if (OpIdx >= Desc.getNumOperands())
199       return nullptr;
200     int RegClass = Desc.OpInfo[OpIdx].RegClass;
201     if (RegClass == -1)
202       return nullptr;
203 
204     return Subtarget->getRegisterInfo()->getRegClass(RegClass);
205   }
206   case AMDGPU::REG_SEQUENCE: {
207     unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
208     const TargetRegisterClass *SuperRC =
209         Subtarget->getRegisterInfo()->getRegClass(RCID);
210 
211     SDValue SubRegOp = N->getOperand(OpNo + 1);
212     unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
213     return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
214                                                               SubRegIdx);
215   }
216   }
217 }
218 
glueCopyToM0(SDNode * N) const219 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N) const {
220   if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS ||
221       cast<MemSDNode>(N)->getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS)
222     return N;
223 
224   const SITargetLowering& Lowering =
225       *static_cast<const SITargetLowering*>(getTargetLowering());
226 
227   // Write max value to m0 before each load operation
228 
229   SDValue M0 = Lowering.copyToM0(*CurDAG, CurDAG->getEntryNode(), SDLoc(N),
230                                  CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
231 
232   SDValue Glue = M0.getValue(1);
233 
234   SmallVector <SDValue, 8> Ops;
235   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
236      Ops.push_back(N->getOperand(i));
237   }
238   Ops.push_back(Glue);
239   CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
240 
241   return N;
242 }
243 
selectSGPRVectorRegClassID(unsigned NumVectorElts)244 static unsigned selectSGPRVectorRegClassID(unsigned NumVectorElts) {
245   switch (NumVectorElts) {
246   case 1:
247     return AMDGPU::SReg_32RegClassID;
248   case 2:
249     return AMDGPU::SReg_64RegClassID;
250   case 4:
251     return AMDGPU::SReg_128RegClassID;
252   case 8:
253     return AMDGPU::SReg_256RegClassID;
254   case 16:
255     return AMDGPU::SReg_512RegClassID;
256   }
257 
258   llvm_unreachable("invalid vector size");
259 }
260 
Select(SDNode * N)261 void AMDGPUDAGToDAGISel::Select(SDNode *N) {
262   unsigned int Opc = N->getOpcode();
263   if (N->isMachineOpcode()) {
264     N->setNodeId(-1);
265     return;   // Already selected.
266   }
267 
268   if (isa<AtomicSDNode>(N) ||
269       (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC))
270     N = glueCopyToM0(N);
271 
272   switch (Opc) {
273   default: break;
274   // We are selecting i64 ADD here instead of custom lower it during
275   // DAG legalization, so we can fold some i64 ADDs used for address
276   // calculation into the LOAD and STORE instructions.
277   case ISD::ADD:
278   case ISD::SUB: {
279     if (N->getValueType(0) != MVT::i64 ||
280         Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
281       break;
282 
283     SelectADD_SUB_I64(N);
284     return;
285   }
286   case ISD::SCALAR_TO_VECTOR:
287   case AMDGPUISD::BUILD_VERTICAL_VECTOR:
288   case ISD::BUILD_VECTOR: {
289     unsigned RegClassID;
290     const AMDGPURegisterInfo *TRI = Subtarget->getRegisterInfo();
291     EVT VT = N->getValueType(0);
292     unsigned NumVectorElts = VT.getVectorNumElements();
293     EVT EltVT = VT.getVectorElementType();
294     assert(EltVT.bitsEq(MVT::i32));
295     if (Subtarget->getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) {
296       RegClassID = selectSGPRVectorRegClassID(NumVectorElts);
297     } else {
298       // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
299       // that adds a 128 bits reg copy when going through TwoAddressInstructions
300       // pass. We want to avoid 128 bits copies as much as possible because they
301       // can't be bundled by our scheduler.
302       switch(NumVectorElts) {
303       case 2: RegClassID = AMDGPU::R600_Reg64RegClassID; break;
304       case 4:
305         if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR)
306           RegClassID = AMDGPU::R600_Reg128VerticalRegClassID;
307         else
308           RegClassID = AMDGPU::R600_Reg128RegClassID;
309         break;
310       default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR");
311       }
312     }
313 
314     SDLoc DL(N);
315     SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
316 
317     if (NumVectorElts == 1) {
318       CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0),
319                            RegClass);
320       return;
321     }
322 
323     assert(NumVectorElts <= 16 && "Vectors with more than 16 elements not "
324                                   "supported yet");
325     // 16 = Max Num Vector Elements
326     // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
327     // 1 = Vector Register Class
328     SmallVector<SDValue, 16 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
329 
330     RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
331     bool IsRegSeq = true;
332     unsigned NOps = N->getNumOperands();
333     for (unsigned i = 0; i < NOps; i++) {
334       // XXX: Why is this here?
335       if (isa<RegisterSDNode>(N->getOperand(i))) {
336         IsRegSeq = false;
337         break;
338       }
339       RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
340       RegSeqArgs[1 + (2 * i) + 1] =
341               CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), DL,
342                                         MVT::i32);
343     }
344 
345     if (NOps != NumVectorElts) {
346       // Fill in the missing undef elements if this was a scalar_to_vector.
347       assert(Opc == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
348 
349       MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
350                                                      DL, EltVT);
351       for (unsigned i = NOps; i < NumVectorElts; ++i) {
352         RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
353         RegSeqArgs[1 + (2 * i) + 1] =
354           CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), DL, MVT::i32);
355       }
356     }
357 
358     if (!IsRegSeq)
359       break;
360     CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs);
361     return;
362   }
363   case ISD::BUILD_PAIR: {
364     SDValue RC, SubReg0, SubReg1;
365     if (Subtarget->getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
366       break;
367     }
368     SDLoc DL(N);
369     if (N->getValueType(0) == MVT::i128) {
370       RC = CurDAG->getTargetConstant(AMDGPU::SReg_128RegClassID, DL, MVT::i32);
371       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
372       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
373     } else if (N->getValueType(0) == MVT::i64) {
374       RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
375       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
376       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
377     } else {
378       llvm_unreachable("Unhandled value type for BUILD_PAIR");
379     }
380     const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
381                             N->getOperand(1), SubReg1 };
382     ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
383                                           N->getValueType(0), Ops));
384     return;
385   }
386 
387   case ISD::Constant:
388   case ISD::ConstantFP: {
389     if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS ||
390         N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N))
391       break;
392 
393     uint64_t Imm;
394     if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N))
395       Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
396     else {
397       ConstantSDNode *C = cast<ConstantSDNode>(N);
398       Imm = C->getZExtValue();
399     }
400 
401     SDLoc DL(N);
402     SDNode *Lo = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
403                                 CurDAG->getConstant(Imm & 0xFFFFFFFF, DL,
404                                                     MVT::i32));
405     SDNode *Hi = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
406                                 CurDAG->getConstant(Imm >> 32, DL, MVT::i32));
407     const SDValue Ops[] = {
408       CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
409       SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
410       SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)
411     };
412 
413     ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
414                                           N->getValueType(0), Ops));
415     return;
416   }
417   case ISD::LOAD:
418   case ISD::STORE: {
419     N = glueCopyToM0(N);
420     break;
421   }
422 
423   case AMDGPUISD::BFE_I32:
424   case AMDGPUISD::BFE_U32: {
425     if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
426       break;
427 
428     // There is a scalar version available, but unlike the vector version which
429     // has a separate operand for the offset and width, the scalar version packs
430     // the width and offset into a single operand. Try to move to the scalar
431     // version if the offsets are constant, so that we can try to keep extended
432     // loads of kernel arguments in SGPRs.
433 
434     // TODO: Technically we could try to pattern match scalar bitshifts of
435     // dynamic values, but it's probably not useful.
436     ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
437     if (!Offset)
438       break;
439 
440     ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
441     if (!Width)
442       break;
443 
444     bool Signed = Opc == AMDGPUISD::BFE_I32;
445 
446     uint32_t OffsetVal = Offset->getZExtValue();
447     uint32_t WidthVal = Width->getZExtValue();
448 
449     ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32,
450                             SDLoc(N), N->getOperand(0), OffsetVal, WidthVal));
451     return;
452   }
453   case AMDGPUISD::DIV_SCALE: {
454     SelectDIV_SCALE(N);
455     return;
456   }
457   case ISD::CopyToReg: {
458     const SITargetLowering& Lowering =
459       *static_cast<const SITargetLowering*>(getTargetLowering());
460     Lowering.legalizeTargetIndependentNode(N, *CurDAG);
461     break;
462   }
463   case ISD::AND:
464   case ISD::SRL:
465   case ISD::SRA:
466   case ISD::SIGN_EXTEND_INREG:
467     if (N->getValueType(0) != MVT::i32 ||
468         Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
469       break;
470 
471     SelectS_BFE(N);
472     return;
473   case ISD::BRCOND:
474     SelectBRCOND(N);
475     return;
476 
477   case AMDGPUISD::ATOMIC_CMP_SWAP:
478     SelectATOMIC_CMP_SWAP(N);
479     return;
480   }
481 
482   SelectCode(N);
483 }
484 
isConstantLoad(const MemSDNode * N,int CbId) const485 bool AMDGPUDAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const {
486   if (!N->readMem())
487     return false;
488   if (CbId == -1)
489     return N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS;
490 
491   return N->getAddressSpace() == AMDGPUAS::CONSTANT_BUFFER_0 + CbId;
492 }
493 
isUniformBr(const SDNode * N) const494 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
495   const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
496   const Instruction *Term = BB->getTerminator();
497   return Term->getMetadata("amdgpu.uniform") ||
498          Term->getMetadata("structurizecfg.uniform");
499 }
500 
getPassName() const501 const char *AMDGPUDAGToDAGISel::getPassName() const {
502   return "AMDGPU DAG->DAG Pattern Instruction Selection";
503 }
504 
505 //===----------------------------------------------------------------------===//
506 // Complex Patterns
507 //===----------------------------------------------------------------------===//
508 
SelectGlobalValueConstantOffset(SDValue Addr,SDValue & IntPtr)509 bool AMDGPUDAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr,
510                                                          SDValue& IntPtr) {
511   if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) {
512     IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr),
513                                        true);
514     return true;
515   }
516   return false;
517 }
518 
SelectGlobalValueVariableOffset(SDValue Addr,SDValue & BaseReg,SDValue & Offset)519 bool AMDGPUDAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr,
520     SDValue& BaseReg, SDValue &Offset) {
521   if (!isa<ConstantSDNode>(Addr)) {
522     BaseReg = Addr;
523     Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true);
524     return true;
525   }
526   return false;
527 }
528 
SelectADDRVTX_READ(SDValue Addr,SDValue & Base,SDValue & Offset)529 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
530                                            SDValue &Offset) {
531   ConstantSDNode *IMMOffset;
532 
533   if (Addr.getOpcode() == ISD::ADD
534       && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))
535       && isInt<16>(IMMOffset->getZExtValue())) {
536 
537       Base = Addr.getOperand(0);
538       Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
539                                          MVT::i32);
540       return true;
541   // If the pointer address is constant, we can move it to the offset field.
542   } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr))
543              && isInt<16>(IMMOffset->getZExtValue())) {
544     Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
545                                   SDLoc(CurDAG->getEntryNode()),
546                                   AMDGPU::ZERO, MVT::i32);
547     Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
548                                        MVT::i32);
549     return true;
550   }
551 
552   // Default case, no offset
553   Base = Addr;
554   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
555   return true;
556 }
557 
SelectADDRIndirect(SDValue Addr,SDValue & Base,SDValue & Offset)558 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
559                                             SDValue &Offset) {
560   ConstantSDNode *C;
561   SDLoc DL(Addr);
562 
563   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
564     Base = CurDAG->getRegister(AMDGPU::INDIRECT_BASE_ADDR, MVT::i32);
565     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
566   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
567             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
568     Base = Addr.getOperand(0);
569     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
570   } else {
571     Base = Addr;
572     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
573   }
574 
575   return true;
576 }
577 
SelectADD_SUB_I64(SDNode * N)578 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
579   SDLoc DL(N);
580   SDValue LHS = N->getOperand(0);
581   SDValue RHS = N->getOperand(1);
582 
583   bool IsAdd = (N->getOpcode() == ISD::ADD);
584 
585   SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
586   SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
587 
588   SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
589                                        DL, MVT::i32, LHS, Sub0);
590   SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
591                                        DL, MVT::i32, LHS, Sub1);
592 
593   SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
594                                        DL, MVT::i32, RHS, Sub0);
595   SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
596                                        DL, MVT::i32, RHS, Sub1);
597 
598   SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue);
599   SDValue AddLoArgs[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
600 
601   unsigned Opc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
602   unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
603 
604   SDNode *AddLo = CurDAG->getMachineNode( Opc, DL, VTList, AddLoArgs);
605   SDValue Carry(AddLo, 1);
606   SDNode *AddHi
607     = CurDAG->getMachineNode(CarryOpc, DL, MVT::i32,
608                              SDValue(Hi0, 0), SDValue(Hi1, 0), Carry);
609 
610   SDValue Args[5] = {
611     CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
612     SDValue(AddLo,0),
613     Sub0,
614     SDValue(AddHi,0),
615     Sub1,
616   };
617   CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, MVT::i64, Args);
618 }
619 
620 // We need to handle this here because tablegen doesn't support matching
621 // instructions with multiple outputs.
SelectDIV_SCALE(SDNode * N)622 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
623   SDLoc SL(N);
624   EVT VT = N->getValueType(0);
625 
626   assert(VT == MVT::f32 || VT == MVT::f64);
627 
628   unsigned Opc
629     = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32;
630 
631   // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp,
632   // omod
633   SDValue Ops[8];
634 
635   SelectVOP3Mods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]);
636   SelectVOP3Mods(N->getOperand(1), Ops[3], Ops[2]);
637   SelectVOP3Mods(N->getOperand(2), Ops[5], Ops[4]);
638   CurDAG->SelectNodeTo(N, Opc, VT, MVT::i1, Ops);
639 }
640 
isDSOffsetLegal(const SDValue & Base,unsigned Offset,unsigned OffsetBits) const641 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(const SDValue &Base, unsigned Offset,
642                                          unsigned OffsetBits) const {
643   if ((OffsetBits == 16 && !isUInt<16>(Offset)) ||
644       (OffsetBits == 8 && !isUInt<8>(Offset)))
645     return false;
646 
647   if (Subtarget->getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS ||
648       Subtarget->unsafeDSOffsetFoldingEnabled())
649     return true;
650 
651   // On Southern Islands instruction with a negative base value and an offset
652   // don't seem to work.
653   return CurDAG->SignBitIsZero(Base);
654 }
655 
SelectDS1Addr1Offset(SDValue Addr,SDValue & Base,SDValue & Offset) const656 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
657                                               SDValue &Offset) const {
658   SDLoc DL(Addr);
659   if (CurDAG->isBaseWithConstantOffset(Addr)) {
660     SDValue N0 = Addr.getOperand(0);
661     SDValue N1 = Addr.getOperand(1);
662     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
663     if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) {
664       // (add n0, c0)
665       Base = N0;
666       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
667       return true;
668     }
669   } else if (Addr.getOpcode() == ISD::SUB) {
670     // sub C, x -> add (sub 0, x), C
671     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
672       int64_t ByteOffset = C->getSExtValue();
673       if (isUInt<16>(ByteOffset)) {
674         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
675 
676         // XXX - This is kind of hacky. Create a dummy sub node so we can check
677         // the known bits in isDSOffsetLegal. We need to emit the selected node
678         // here, so this is thrown away.
679         SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
680                                       Zero, Addr.getOperand(1));
681 
682         if (isDSOffsetLegal(Sub, ByteOffset, 16)) {
683           MachineSDNode *MachineSub
684             = CurDAG->getMachineNode(AMDGPU::V_SUB_I32_e32, DL, MVT::i32,
685                                      Zero, Addr.getOperand(1));
686 
687           Base = SDValue(MachineSub, 0);
688           Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16);
689           return true;
690         }
691       }
692     }
693   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
694     // If we have a constant address, prefer to put the constant into the
695     // offset. This can save moves to load the constant address since multiple
696     // operations can share the zero base address register, and enables merging
697     // into read2 / write2 instructions.
698 
699     SDLoc DL(Addr);
700 
701     if (isUInt<16>(CAddr->getZExtValue())) {
702       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
703       MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
704                                  DL, MVT::i32, Zero);
705       Base = SDValue(MovZero, 0);
706       Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
707       return true;
708     }
709   }
710 
711   // default case
712   Base = Addr;
713   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
714   return true;
715 }
716 
717 // TODO: If offset is too big, put low 16-bit into offset.
SelectDS64Bit4ByteAligned(SDValue Addr,SDValue & Base,SDValue & Offset0,SDValue & Offset1) const718 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
719                                                    SDValue &Offset0,
720                                                    SDValue &Offset1) const {
721   SDLoc DL(Addr);
722 
723   if (CurDAG->isBaseWithConstantOffset(Addr)) {
724     SDValue N0 = Addr.getOperand(0);
725     SDValue N1 = Addr.getOperand(1);
726     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
727     unsigned DWordOffset0 = C1->getZExtValue() / 4;
728     unsigned DWordOffset1 = DWordOffset0 + 1;
729     // (add n0, c0)
730     if (isDSOffsetLegal(N0, DWordOffset1, 8)) {
731       Base = N0;
732       Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
733       Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
734       return true;
735     }
736   } else if (Addr.getOpcode() == ISD::SUB) {
737     // sub C, x -> add (sub 0, x), C
738     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
739       unsigned DWordOffset0 = C->getZExtValue() / 4;
740       unsigned DWordOffset1 = DWordOffset0 + 1;
741 
742       if (isUInt<8>(DWordOffset0)) {
743         SDLoc DL(Addr);
744         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
745 
746         // XXX - This is kind of hacky. Create a dummy sub node so we can check
747         // the known bits in isDSOffsetLegal. We need to emit the selected node
748         // here, so this is thrown away.
749         SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
750                                       Zero, Addr.getOperand(1));
751 
752         if (isDSOffsetLegal(Sub, DWordOffset1, 8)) {
753           MachineSDNode *MachineSub
754             = CurDAG->getMachineNode(AMDGPU::V_SUB_I32_e32, DL, MVT::i32,
755                                      Zero, Addr.getOperand(1));
756 
757           Base = SDValue(MachineSub, 0);
758           Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
759           Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
760           return true;
761         }
762       }
763     }
764   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
765     unsigned DWordOffset0 = CAddr->getZExtValue() / 4;
766     unsigned DWordOffset1 = DWordOffset0 + 1;
767     assert(4 * DWordOffset0 == CAddr->getZExtValue());
768 
769     if (isUInt<8>(DWordOffset0) && isUInt<8>(DWordOffset1)) {
770       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
771       MachineSDNode *MovZero
772         = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
773                                  DL, MVT::i32, Zero);
774       Base = SDValue(MovZero, 0);
775       Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
776       Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
777       return true;
778     }
779   }
780 
781   // default case
782   Base = Addr;
783   Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
784   Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
785   return true;
786 }
787 
isLegalMUBUFImmOffset(const ConstantSDNode * Imm)788 static bool isLegalMUBUFImmOffset(const ConstantSDNode *Imm) {
789   return isUInt<12>(Imm->getZExtValue());
790 }
791 
SelectMUBUF(SDValue Addr,SDValue & Ptr,SDValue & VAddr,SDValue & SOffset,SDValue & Offset,SDValue & Offen,SDValue & Idxen,SDValue & Addr64,SDValue & GLC,SDValue & SLC,SDValue & TFE) const792 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
793                                      SDValue &VAddr, SDValue &SOffset,
794                                      SDValue &Offset, SDValue &Offen,
795                                      SDValue &Idxen, SDValue &Addr64,
796                                      SDValue &GLC, SDValue &SLC,
797                                      SDValue &TFE) const {
798   // Subtarget prefers to use flat instruction
799   if (Subtarget->useFlatForGlobal())
800     return false;
801 
802   SDLoc DL(Addr);
803 
804   if (!GLC.getNode())
805     GLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
806   if (!SLC.getNode())
807     SLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
808   TFE = CurDAG->getTargetConstant(0, DL, MVT::i1);
809 
810   Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
811   Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
812   Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
813   SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
814 
815   if (CurDAG->isBaseWithConstantOffset(Addr)) {
816     SDValue N0 = Addr.getOperand(0);
817     SDValue N1 = Addr.getOperand(1);
818     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
819 
820     if (N0.getOpcode() == ISD::ADD) {
821       // (add (add N2, N3), C1) -> addr64
822       SDValue N2 = N0.getOperand(0);
823       SDValue N3 = N0.getOperand(1);
824       Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
825       Ptr = N2;
826       VAddr = N3;
827     } else {
828 
829       // (add N0, C1) -> offset
830       VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
831       Ptr = N0;
832     }
833 
834     if (isLegalMUBUFImmOffset(C1)) {
835       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
836       return true;
837     }
838 
839     if (isUInt<32>(C1->getZExtValue())) {
840       // Illegal offset, store it in soffset.
841       Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
842       SOffset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
843                    CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
844                         0);
845       return true;
846     }
847   }
848 
849   if (Addr.getOpcode() == ISD::ADD) {
850     // (add N0, N1) -> addr64
851     SDValue N0 = Addr.getOperand(0);
852     SDValue N1 = Addr.getOperand(1);
853     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
854     Ptr = N0;
855     VAddr = N1;
856     Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
857     return true;
858   }
859 
860   // default case -> offset
861   VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
862   Ptr = Addr;
863   Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
864 
865   return true;
866 }
867 
SelectMUBUFAddr64(SDValue Addr,SDValue & SRsrc,SDValue & VAddr,SDValue & SOffset,SDValue & Offset,SDValue & GLC,SDValue & SLC,SDValue & TFE) const868 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
869                                            SDValue &VAddr, SDValue &SOffset,
870                                            SDValue &Offset, SDValue &GLC,
871                                            SDValue &SLC, SDValue &TFE) const {
872   SDValue Ptr, Offen, Idxen, Addr64;
873 
874   // addr64 bit was removed for volcanic islands.
875   if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
876     return false;
877 
878   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
879               GLC, SLC, TFE))
880     return false;
881 
882   ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
883   if (C->getSExtValue()) {
884     SDLoc DL(Addr);
885 
886     const SITargetLowering& Lowering =
887       *static_cast<const SITargetLowering*>(getTargetLowering());
888 
889     SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
890     return true;
891   }
892 
893   return false;
894 }
895 
SelectMUBUFAddr64(SDValue Addr,SDValue & SRsrc,SDValue & VAddr,SDValue & SOffset,SDValue & Offset,SDValue & SLC) const896 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
897                                            SDValue &VAddr, SDValue &SOffset,
898                                            SDValue &Offset,
899                                            SDValue &SLC) const {
900   SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1);
901   SDValue GLC, TFE;
902 
903   return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE);
904 }
905 
SelectMUBUFScratch(SDValue Addr,SDValue & Rsrc,SDValue & VAddr,SDValue & SOffset,SDValue & ImmOffset) const906 bool AMDGPUDAGToDAGISel::SelectMUBUFScratch(SDValue Addr, SDValue &Rsrc,
907                                             SDValue &VAddr, SDValue &SOffset,
908                                             SDValue &ImmOffset) const {
909 
910   SDLoc DL(Addr);
911   MachineFunction &MF = CurDAG->getMachineFunction();
912   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
913 
914   Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
915   SOffset = CurDAG->getRegister(Info->getScratchWaveOffsetReg(), MVT::i32);
916 
917   // (add n0, c1)
918   if (CurDAG->isBaseWithConstantOffset(Addr)) {
919     SDValue N0 = Addr.getOperand(0);
920     SDValue N1 = Addr.getOperand(1);
921 
922     // Offsets in vaddr must be positive.
923     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
924     if (isLegalMUBUFImmOffset(C1)) {
925       VAddr = N0;
926       ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
927       return true;
928     }
929   }
930 
931   // (node)
932   VAddr = Addr;
933   ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
934   return true;
935 }
936 
SelectMUBUFOffset(SDValue Addr,SDValue & SRsrc,SDValue & SOffset,SDValue & Offset,SDValue & GLC,SDValue & SLC,SDValue & TFE) const937 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
938                                            SDValue &SOffset, SDValue &Offset,
939                                            SDValue &GLC, SDValue &SLC,
940                                            SDValue &TFE) const {
941   SDValue Ptr, VAddr, Offen, Idxen, Addr64;
942   const SIInstrInfo *TII =
943     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
944 
945   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
946               GLC, SLC, TFE))
947     return false;
948 
949   if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
950       !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
951       !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
952     uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
953                     APInt::getAllOnesValue(32).getZExtValue(); // Size
954     SDLoc DL(Addr);
955 
956     const SITargetLowering& Lowering =
957       *static_cast<const SITargetLowering*>(getTargetLowering());
958 
959     SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
960     return true;
961   }
962   return false;
963 }
964 
SelectMUBUFOffset(SDValue Addr,SDValue & SRsrc,SDValue & Soffset,SDValue & Offset) const965 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
966                                            SDValue &Soffset, SDValue &Offset
967                                            ) const {
968   SDValue GLC, SLC, TFE;
969 
970   return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE);
971 }
SelectMUBUFOffset(SDValue Addr,SDValue & SRsrc,SDValue & Soffset,SDValue & Offset,SDValue & SLC) const972 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
973                                            SDValue &Soffset, SDValue &Offset,
974                                            SDValue &SLC) const {
975   SDValue GLC, TFE;
976 
977   return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE);
978 }
979 
SelectMUBUFConstant(SDValue Constant,SDValue & SOffset,SDValue & ImmOffset) const980 bool AMDGPUDAGToDAGISel::SelectMUBUFConstant(SDValue Constant,
981                                              SDValue &SOffset,
982                                              SDValue &ImmOffset) const {
983   SDLoc DL(Constant);
984   uint32_t Imm = cast<ConstantSDNode>(Constant)->getZExtValue();
985   uint32_t Overflow = 0;
986 
987   if (Imm >= 4096) {
988     if (Imm <= 4095 + 64) {
989       // Use an SOffset inline constant for 1..64
990       Overflow = Imm - 4095;
991       Imm = 4095;
992     } else {
993       // Try to keep the same value in SOffset for adjacent loads, so that
994       // the corresponding register contents can be re-used.
995       //
996       // Load values with all low-bits set into SOffset, so that a larger
997       // range of values can be covered using s_movk_i32
998       uint32_t High = (Imm + 1) & ~4095;
999       uint32_t Low = (Imm + 1) & 4095;
1000       Imm = Low;
1001       Overflow = High - 1;
1002     }
1003   }
1004 
1005   // There is a hardware bug in SI and CI which prevents address clamping in
1006   // MUBUF instructions from working correctly with SOffsets. The immediate
1007   // offset is unaffected.
1008   if (Overflow > 0 &&
1009       Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
1010     return false;
1011 
1012   ImmOffset = CurDAG->getTargetConstant(Imm, DL, MVT::i16);
1013 
1014   if (Overflow <= 64)
1015     SOffset = CurDAG->getTargetConstant(Overflow, DL, MVT::i32);
1016   else
1017     SOffset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
1018                       CurDAG->getTargetConstant(Overflow, DL, MVT::i32)),
1019                       0);
1020 
1021   return true;
1022 }
1023 
SelectMUBUFIntrinsicOffset(SDValue Offset,SDValue & SOffset,SDValue & ImmOffset) const1024 bool AMDGPUDAGToDAGISel::SelectMUBUFIntrinsicOffset(SDValue Offset,
1025                                                     SDValue &SOffset,
1026                                                     SDValue &ImmOffset) const {
1027   SDLoc DL(Offset);
1028 
1029   if (!isa<ConstantSDNode>(Offset))
1030     return false;
1031 
1032   return SelectMUBUFConstant(Offset, SOffset, ImmOffset);
1033 }
1034 
SelectMUBUFIntrinsicVOffset(SDValue Offset,SDValue & SOffset,SDValue & ImmOffset,SDValue & VOffset) const1035 bool AMDGPUDAGToDAGISel::SelectMUBUFIntrinsicVOffset(SDValue Offset,
1036                                                      SDValue &SOffset,
1037                                                      SDValue &ImmOffset,
1038                                                      SDValue &VOffset) const {
1039   SDLoc DL(Offset);
1040 
1041   // Don't generate an unnecessary voffset for constant offsets.
1042   if (isa<ConstantSDNode>(Offset)) {
1043     SDValue Tmp1, Tmp2;
1044 
1045     // When necessary, use a voffset in <= CI anyway to work around a hardware
1046     // bug.
1047     if (Subtarget->getGeneration() > AMDGPUSubtarget::SEA_ISLANDS ||
1048         SelectMUBUFConstant(Offset, Tmp1, Tmp2))
1049       return false;
1050   }
1051 
1052   if (CurDAG->isBaseWithConstantOffset(Offset)) {
1053     SDValue N0 = Offset.getOperand(0);
1054     SDValue N1 = Offset.getOperand(1);
1055     if (cast<ConstantSDNode>(N1)->getSExtValue() >= 0 &&
1056         SelectMUBUFConstant(N1, SOffset, ImmOffset)) {
1057       VOffset = N0;
1058       return true;
1059     }
1060   }
1061 
1062   SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1063   ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1064   VOffset = Offset;
1065 
1066   return true;
1067 }
1068 
SelectFlat(SDValue Addr,SDValue & VAddr,SDValue & SLC,SDValue & TFE) const1069 bool AMDGPUDAGToDAGISel::SelectFlat(SDValue Addr,
1070                                     SDValue &VAddr,
1071                                     SDValue &SLC,
1072                                     SDValue &TFE) const {
1073   VAddr = Addr;
1074   TFE = SLC = CurDAG->getTargetConstant(0, SDLoc(), MVT::i1);
1075   return true;
1076 }
1077 
1078 ///
1079 /// \param EncodedOffset This is the immediate value that will be encoded
1080 ///        directly into the instruction.  On SI/CI the \p EncodedOffset
1081 ///        will be in units of dwords and on VI+ it will be units of bytes.
isLegalSMRDImmOffset(const AMDGPUSubtarget * ST,int64_t EncodedOffset)1082 static bool isLegalSMRDImmOffset(const AMDGPUSubtarget *ST,
1083                                  int64_t EncodedOffset) {
1084   return ST->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS ?
1085      isUInt<8>(EncodedOffset) : isUInt<20>(EncodedOffset);
1086 }
1087 
SelectSMRDOffset(SDValue ByteOffsetNode,SDValue & Offset,bool & Imm) const1088 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
1089                                           SDValue &Offset, bool &Imm) const {
1090 
1091   // FIXME: Handle non-constant offsets.
1092   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
1093   if (!C)
1094     return false;
1095 
1096   SDLoc SL(ByteOffsetNode);
1097   AMDGPUSubtarget::Generation Gen = Subtarget->getGeneration();
1098   int64_t ByteOffset = C->getSExtValue();
1099   int64_t EncodedOffset = Gen < AMDGPUSubtarget::VOLCANIC_ISLANDS ?
1100       ByteOffset >> 2 : ByteOffset;
1101 
1102   if (isLegalSMRDImmOffset(Subtarget, EncodedOffset)) {
1103     Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
1104     Imm = true;
1105     return true;
1106   }
1107 
1108   if (!isUInt<32>(EncodedOffset) || !isUInt<32>(ByteOffset))
1109     return false;
1110 
1111   if (Gen == AMDGPUSubtarget::SEA_ISLANDS && isUInt<32>(EncodedOffset)) {
1112     // 32-bit Immediates are supported on Sea Islands.
1113     Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
1114   } else {
1115     SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
1116     Offset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32,
1117                                             C32Bit), 0);
1118   }
1119   Imm = false;
1120   return true;
1121 }
1122 
SelectSMRD(SDValue Addr,SDValue & SBase,SDValue & Offset,bool & Imm) const1123 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
1124                                      SDValue &Offset, bool &Imm) const {
1125 
1126   SDLoc SL(Addr);
1127   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1128     SDValue N0 = Addr.getOperand(0);
1129     SDValue N1 = Addr.getOperand(1);
1130 
1131     if (SelectSMRDOffset(N1, Offset, Imm)) {
1132       SBase = N0;
1133       return true;
1134     }
1135   }
1136   SBase = Addr;
1137   Offset = CurDAG->getTargetConstant(0, SL, MVT::i32);
1138   Imm = true;
1139   return true;
1140 }
1141 
SelectSMRDImm(SDValue Addr,SDValue & SBase,SDValue & Offset) const1142 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
1143                                        SDValue &Offset) const {
1144   bool Imm;
1145   return SelectSMRD(Addr, SBase, Offset, Imm) && Imm;
1146 }
1147 
SelectSMRDImm32(SDValue Addr,SDValue & SBase,SDValue & Offset) const1148 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
1149                                          SDValue &Offset) const {
1150 
1151   if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS)
1152     return false;
1153 
1154   bool Imm;
1155   if (!SelectSMRD(Addr, SBase, Offset, Imm))
1156     return false;
1157 
1158   return !Imm && isa<ConstantSDNode>(Offset);
1159 }
1160 
SelectSMRDSgpr(SDValue Addr,SDValue & SBase,SDValue & Offset) const1161 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
1162                                         SDValue &Offset) const {
1163   bool Imm;
1164   return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm &&
1165          !isa<ConstantSDNode>(Offset);
1166 }
1167 
SelectSMRDBufferImm(SDValue Addr,SDValue & Offset) const1168 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr,
1169                                              SDValue &Offset) const {
1170   bool Imm;
1171   return SelectSMRDOffset(Addr, Offset, Imm) && Imm;
1172 }
1173 
SelectSMRDBufferImm32(SDValue Addr,SDValue & Offset) const1174 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr,
1175                                                SDValue &Offset) const {
1176   if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS)
1177     return false;
1178 
1179   bool Imm;
1180   if (!SelectSMRDOffset(Addr, Offset, Imm))
1181     return false;
1182 
1183   return !Imm && isa<ConstantSDNode>(Offset);
1184 }
1185 
SelectSMRDBufferSgpr(SDValue Addr,SDValue & Offset) const1186 bool AMDGPUDAGToDAGISel::SelectSMRDBufferSgpr(SDValue Addr,
1187                                               SDValue &Offset) const {
1188   bool Imm;
1189   return SelectSMRDOffset(Addr, Offset, Imm) && !Imm &&
1190          !isa<ConstantSDNode>(Offset);
1191 }
1192 
SelectMOVRELOffset(SDValue Index,SDValue & Base,SDValue & Offset) const1193 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
1194                                             SDValue &Base,
1195                                             SDValue &Offset) const {
1196   SDLoc DL(Index);
1197 
1198   if (CurDAG->isBaseWithConstantOffset(Index)) {
1199     SDValue N0 = Index.getOperand(0);
1200     SDValue N1 = Index.getOperand(1);
1201     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1202 
1203     // (add n0, c0)
1204     Base = N0;
1205     Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
1206     return true;
1207   }
1208 
1209   if (isa<ConstantSDNode>(Index))
1210     return false;
1211 
1212   Base = Index;
1213   Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1214   return true;
1215 }
1216 
getS_BFE(unsigned Opcode,const SDLoc & DL,SDValue Val,uint32_t Offset,uint32_t Width)1217 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL,
1218                                      SDValue Val, uint32_t Offset,
1219                                      uint32_t Width) {
1220   // Transformation function, pack the offset and width of a BFE into
1221   // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
1222   // source, bits [5:0] contain the offset and bits [22:16] the width.
1223   uint32_t PackedVal = Offset | (Width << 16);
1224   SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
1225 
1226   return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
1227 }
1228 
SelectS_BFEFromShifts(SDNode * N)1229 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
1230   // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
1231   // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
1232   // Predicate: 0 < b <= c < 32
1233 
1234   const SDValue &Shl = N->getOperand(0);
1235   ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
1236   ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
1237 
1238   if (B && C) {
1239     uint32_t BVal = B->getZExtValue();
1240     uint32_t CVal = C->getZExtValue();
1241 
1242     if (0 < BVal && BVal <= CVal && CVal < 32) {
1243       bool Signed = N->getOpcode() == ISD::SRA;
1244       unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
1245 
1246       ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal,
1247                               32 - CVal));
1248       return;
1249     }
1250   }
1251   SelectCode(N);
1252 }
1253 
SelectS_BFE(SDNode * N)1254 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
1255   switch (N->getOpcode()) {
1256   case ISD::AND:
1257     if (N->getOperand(0).getOpcode() == ISD::SRL) {
1258       // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
1259       // Predicate: isMask(mask)
1260       const SDValue &Srl = N->getOperand(0);
1261       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
1262       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
1263 
1264       if (Shift && Mask) {
1265         uint32_t ShiftVal = Shift->getZExtValue();
1266         uint32_t MaskVal = Mask->getZExtValue();
1267 
1268         if (isMask_32(MaskVal)) {
1269           uint32_t WidthVal = countPopulation(MaskVal);
1270 
1271           ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
1272                                   Srl.getOperand(0), ShiftVal, WidthVal));
1273           return;
1274         }
1275       }
1276     }
1277     break;
1278   case ISD::SRL:
1279     if (N->getOperand(0).getOpcode() == ISD::AND) {
1280       // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
1281       // Predicate: isMask(mask >> b)
1282       const SDValue &And = N->getOperand(0);
1283       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
1284       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
1285 
1286       if (Shift && Mask) {
1287         uint32_t ShiftVal = Shift->getZExtValue();
1288         uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
1289 
1290         if (isMask_32(MaskVal)) {
1291           uint32_t WidthVal = countPopulation(MaskVal);
1292 
1293           ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
1294                                   And.getOperand(0), ShiftVal, WidthVal));
1295           return;
1296         }
1297       }
1298     } else if (N->getOperand(0).getOpcode() == ISD::SHL) {
1299       SelectS_BFEFromShifts(N);
1300       return;
1301     }
1302     break;
1303   case ISD::SRA:
1304     if (N->getOperand(0).getOpcode() == ISD::SHL) {
1305       SelectS_BFEFromShifts(N);
1306       return;
1307     }
1308     break;
1309 
1310   case ISD::SIGN_EXTEND_INREG: {
1311     // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8
1312     SDValue Src = N->getOperand(0);
1313     if (Src.getOpcode() != ISD::SRL)
1314       break;
1315 
1316     const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
1317     if (!Amt)
1318       break;
1319 
1320     unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
1321     ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0),
1322                             Amt->getZExtValue(), Width));
1323     return;
1324   }
1325   }
1326 
1327   SelectCode(N);
1328 }
1329 
SelectBRCOND(SDNode * N)1330 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
1331   SDValue Cond = N->getOperand(1);
1332 
1333   if (isCBranchSCC(N)) {
1334     // This brcond will use S_CBRANCH_SCC*, so let tablegen handle it.
1335     SelectCode(N);
1336     return;
1337   }
1338 
1339   // The result of VOPC instructions is or'd against ~EXEC before it is
1340   // written to vcc or another SGPR.  This means that the value '1' is always
1341   // written to the corresponding bit for results that are masked.  In order
1342   // to correctly check against vccz, we need to and VCC with the EXEC
1343   // register in order to clear the value from the masked bits.
1344 
1345   SDLoc SL(N);
1346 
1347   SDNode *MaskedCond =
1348         CurDAG->getMachineNode(AMDGPU::S_AND_B64, SL, MVT::i1,
1349                                CurDAG->getRegister(AMDGPU::EXEC, MVT::i1),
1350                                Cond);
1351   SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, AMDGPU::VCC,
1352                                      SDValue(MaskedCond, 0),
1353                                      SDValue()); // Passing SDValue() adds a
1354                                                  // glue output.
1355   CurDAG->SelectNodeTo(N, AMDGPU::S_CBRANCH_VCCNZ, MVT::Other,
1356                        N->getOperand(2), // Basic Block
1357                        VCC.getValue(0),  // Chain
1358                        VCC.getValue(1)); // Glue
1359   return;
1360 }
1361 
1362 // This is here because there isn't a way to use the generated sub0_sub1 as the
1363 // subreg index to EXTRACT_SUBREG in tablegen.
SelectATOMIC_CMP_SWAP(SDNode * N)1364 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
1365   MemSDNode *Mem = cast<MemSDNode>(N);
1366   unsigned AS = Mem->getAddressSpace();
1367   if (AS == AMDGPUAS::FLAT_ADDRESS) {
1368     SelectCode(N);
1369     return;
1370   }
1371 
1372   MVT VT = N->getSimpleValueType(0);
1373   bool Is32 = (VT == MVT::i32);
1374   SDLoc SL(N);
1375 
1376   MachineSDNode *CmpSwap = nullptr;
1377   if (Subtarget->hasAddr64()) {
1378     SDValue SRsrc, VAddr, SOffset, Offset, GLC, SLC;
1379 
1380     if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset, SLC)) {
1381       unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_RTN_ADDR64 :
1382         AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_RTN_ADDR64;
1383       SDValue CmpVal = Mem->getOperand(2);
1384 
1385       // XXX - Do we care about glue operands?
1386 
1387       SDValue Ops[] = {
1388         CmpVal, VAddr, SRsrc, SOffset, Offset, SLC, Mem->getChain()
1389       };
1390 
1391       CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
1392     }
1393   }
1394 
1395   if (!CmpSwap) {
1396     SDValue SRsrc, SOffset, Offset, SLC;
1397     if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset, SLC)) {
1398       unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_RTN_OFFSET :
1399         AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_RTN_OFFSET;
1400 
1401       SDValue CmpVal = Mem->getOperand(2);
1402       SDValue Ops[] = {
1403         CmpVal, SRsrc, SOffset, Offset, SLC, Mem->getChain()
1404       };
1405 
1406       CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
1407     }
1408   }
1409 
1410   if (!CmpSwap) {
1411     SelectCode(N);
1412     return;
1413   }
1414 
1415   MachineSDNode::mmo_iterator MMOs = MF->allocateMemRefsArray(1);
1416   *MMOs = Mem->getMemOperand();
1417   CmpSwap->setMemRefs(MMOs, MMOs + 1);
1418 
1419   unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1;
1420   SDValue Extract
1421     = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0));
1422 
1423   ReplaceUses(SDValue(N, 0), Extract);
1424   ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1));
1425   CurDAG->RemoveDeadNode(N);
1426 }
1427 
SelectVOP3Mods(SDValue In,SDValue & Src,SDValue & SrcMods) const1428 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
1429                                         SDValue &SrcMods) const {
1430 
1431   unsigned Mods = 0;
1432 
1433   Src = In;
1434 
1435   if (Src.getOpcode() == ISD::FNEG) {
1436     Mods |= SISrcMods::NEG;
1437     Src = Src.getOperand(0);
1438   }
1439 
1440   if (Src.getOpcode() == ISD::FABS) {
1441     Mods |= SISrcMods::ABS;
1442     Src = Src.getOperand(0);
1443   }
1444 
1445   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
1446 
1447   return true;
1448 }
1449 
SelectVOP3NoMods(SDValue In,SDValue & Src,SDValue & SrcMods) const1450 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src,
1451                                          SDValue &SrcMods) const {
1452   bool Res = SelectVOP3Mods(In, Src, SrcMods);
1453   return Res && cast<ConstantSDNode>(SrcMods)->isNullValue();
1454 }
1455 
SelectVOP3Mods0(SDValue In,SDValue & Src,SDValue & SrcMods,SDValue & Clamp,SDValue & Omod) const1456 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
1457                                          SDValue &SrcMods, SDValue &Clamp,
1458                                          SDValue &Omod) const {
1459   SDLoc DL(In);
1460   // FIXME: Handle Clamp and Omod
1461   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i32);
1462   Omod = CurDAG->getTargetConstant(0, DL, MVT::i32);
1463 
1464   return SelectVOP3Mods(In, Src, SrcMods);
1465 }
1466 
SelectVOP3NoMods0(SDValue In,SDValue & Src,SDValue & SrcMods,SDValue & Clamp,SDValue & Omod) const1467 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods0(SDValue In, SDValue &Src,
1468                                            SDValue &SrcMods, SDValue &Clamp,
1469                                            SDValue &Omod) const {
1470   bool Res = SelectVOP3Mods0(In, Src, SrcMods, Clamp, Omod);
1471 
1472   return Res && cast<ConstantSDNode>(SrcMods)->isNullValue() &&
1473                 cast<ConstantSDNode>(Clamp)->isNullValue() &&
1474                 cast<ConstantSDNode>(Omod)->isNullValue();
1475 }
1476 
SelectVOP3Mods0Clamp(SDValue In,SDValue & Src,SDValue & SrcMods,SDValue & Omod) const1477 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp(SDValue In, SDValue &Src,
1478                                               SDValue &SrcMods,
1479                                               SDValue &Omod) const {
1480   // FIXME: Handle Omod
1481   Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
1482 
1483   return SelectVOP3Mods(In, Src, SrcMods);
1484 }
1485 
SelectVOP3Mods0Clamp0OMod(SDValue In,SDValue & Src,SDValue & SrcMods,SDValue & Clamp,SDValue & Omod) const1486 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src,
1487                                                    SDValue &SrcMods,
1488                                                    SDValue &Clamp,
1489                                                    SDValue &Omod) const {
1490   Clamp = Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
1491   return SelectVOP3Mods(In, Src, SrcMods);
1492 }
1493 
PreprocessISelDAG()1494 void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
1495   MachineFrameInfo *MFI = CurDAG->getMachineFunction().getFrameInfo();
1496 
1497   // Handle the perverse case where a frame index is being stored. We don't
1498   // want to see multiple frame index operands on the same instruction since
1499   // it complicates things and violates some assumptions about frame index
1500   // lowering.
1501   for (int I = MFI->getObjectIndexBegin(), E = MFI->getObjectIndexEnd();
1502        I != E; ++I) {
1503     SDValue FI = CurDAG->getTargetFrameIndex(I, MVT::i32);
1504 
1505     // It's possible that we have a frame index defined in the function that
1506     // isn't used in this block.
1507     if (FI.use_empty())
1508       continue;
1509 
1510     // Skip over the AssertZext inserted during lowering.
1511     SDValue EffectiveFI = FI;
1512     auto It = FI->use_begin();
1513     if (It->getOpcode() == ISD::AssertZext && FI->hasOneUse()) {
1514       EffectiveFI = SDValue(*It, 0);
1515       It = EffectiveFI->use_begin();
1516     }
1517 
1518     for (auto It = EffectiveFI->use_begin(); !It.atEnd(); ) {
1519       SDUse &Use = It.getUse();
1520       SDNode *User = Use.getUser();
1521       unsigned OpIdx = It.getOperandNo();
1522       ++It;
1523 
1524       if (MemSDNode *M = dyn_cast<MemSDNode>(User)) {
1525         unsigned PtrIdx = M->getOpcode() == ISD::STORE ? 2 : 1;
1526         if (OpIdx == PtrIdx)
1527           continue;
1528 
1529         unsigned OpN = M->getNumOperands();
1530         SDValue NewOps[8];
1531 
1532         assert(OpN < array_lengthof(NewOps));
1533         for (unsigned Op = 0; Op != OpN; ++Op) {
1534           if (Op != OpIdx) {
1535             NewOps[Op] = M->getOperand(Op);
1536             continue;
1537           }
1538 
1539           MachineSDNode *Mov = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1540                                                       SDLoc(M), MVT::i32, FI);
1541           NewOps[Op] = SDValue(Mov, 0);
1542         }
1543 
1544         CurDAG->UpdateNodeOperands(M, makeArrayRef(NewOps, OpN));
1545       }
1546     }
1547   }
1548 }
1549 
PostprocessISelDAG()1550 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
1551   const AMDGPUTargetLowering& Lowering =
1552     *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
1553   bool IsModified = false;
1554   do {
1555     IsModified = false;
1556     // Go over all selected nodes and try to fold them a bit more
1557     for (SDNode &Node : CurDAG->allnodes()) {
1558       MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(&Node);
1559       if (!MachineNode)
1560         continue;
1561 
1562       SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
1563       if (ResNode != &Node) {
1564         ReplaceUses(&Node, ResNode);
1565         IsModified = true;
1566       }
1567     }
1568     CurDAG->RemoveDeadNodes();
1569   } while (IsModified);
1570 }
1571