1 /*
2 * Copyright 2020 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22 #include "codegen/nv50_ir.h"
23 #include "codegen/nv50_ir_build_util.h"
24
25 #include "codegen/nv50_ir_target_nvc0.h"
26 #include "codegen/nv50_ir_lowering_gv100.h"
27
28 #include <limits>
29
30 namespace nv50_ir {
31
32 bool
handleCMP(Instruction * i)33 GV100LegalizeSSA::handleCMP(Instruction *i)
34 {
35 Value *pred = bld.getSSA(1, FILE_PREDICATE);
36
37 bld.mkCmp(OP_SET, reverseCondCode(i->asCmp()->setCond), TYPE_U8, pred,
38 i->sType, bld.mkImm(0), i->getSrc(2))->ftz = i->ftz;
39 bld.mkOp3(OP_SELP, TYPE_U32, i->getDef(0), i->getSrc(0), i->getSrc(1), pred);
40 return true;
41 }
42
43 // NIR deals with most of these for us, but codegen generates more in pointer
44 // calculations from other lowering passes.
45 bool
handleIADD64(Instruction * i)46 GV100LegalizeSSA::handleIADD64(Instruction *i)
47 {
48 Value *carry = bld.getSSA(1, FILE_PREDICATE);
49 Value *def[2] = { bld.getSSA(), bld.getSSA() };
50 Value *src[2][2];
51
52 for (int s = 0; s < 2; s++) {
53 if (i->getSrc(s)->reg.size == 8) {
54 bld.mkSplit(src[s], 4, i->getSrc(s));
55 } else {
56 src[s][0] = i->getSrc(s);
57 src[s][1] = bld.mkImm(0);
58 }
59 }
60
61 bld.mkOp2(OP_ADD, TYPE_U32, def[0], src[0][0], src[1][0])->
62 setFlagsDef(1, carry);
63 bld.mkOp2(OP_ADD, TYPE_U32, def[1], src[0][1], src[1][1])->
64 setFlagsSrc(2, carry);
65 bld.mkOp2(OP_MERGE, i->dType, i->getDef(0), def[0], def[1]);
66 return true;
67 }
68
69 bool
handleIMAD_HIGH(Instruction * i)70 GV100LegalizeSSA::handleIMAD_HIGH(Instruction *i)
71 {
72 Value *def = bld.getSSA(8), *defs[2];
73 Value *src2;
74
75 if (i->srcExists(2) &&
76 (!i->getSrc(2)->asImm() || i->getSrc(2)->asImm()->reg.data.u32)) {
77 Value *src2s[2] = { bld.getSSA(), bld.getSSA() };
78 bld.mkMov(src2s[0], bld.mkImm(0));
79 bld.mkMov(src2s[1], i->getSrc(2));
80 src2 = bld.mkOp2(OP_MERGE, TYPE_U64, bld.getSSA(8), src2s[0], src2s[1])->getDef(0);
81 } else {
82 src2 = bld.mkImm(0);
83 }
84
85 bld.mkOp3(OP_MAD, isSignedType(i->sType) ? TYPE_S64 : TYPE_U64, def,
86 i->getSrc(0), i->getSrc(1), src2);
87
88 bld.mkSplit(defs, 4, def);
89 i->def(0).replace(defs[1], false);
90 return true;
91 }
92
93 // XXX: We should be able to do this in GV100LoweringPass, but codegen messes
94 // up somehow and swaps the condcode without swapping the sources.
95 // - tests/spec/glsl-1.50/execution/geometry/primitive-id-in.shader_test
96 bool
handleIMNMX(Instruction * i)97 GV100LegalizeSSA::handleIMNMX(Instruction *i)
98 {
99 Value *pred = bld.getSSA(1, FILE_PREDICATE);
100
101 bld.mkCmp(OP_SET, (i->op == OP_MIN) ? CC_LT : CC_GT, i->dType, pred,
102 i->sType, i->getSrc(0), i->getSrc(1));
103 bld.mkOp3(OP_SELP, i->dType, i->getDef(0), i->getSrc(0), i->getSrc(1), pred);
104 return true;
105 }
106
107 bool
handleIMUL(Instruction * i)108 GV100LegalizeSSA::handleIMUL(Instruction *i)
109 {
110 if (i->subOp == NV50_IR_SUBOP_MUL_HIGH)
111 return handleIMAD_HIGH(i);
112
113 bld.mkOp3(OP_MAD, i->dType, i->getDef(0), i->getSrc(0), i->getSrc(1),
114 bld.mkImm(0));
115 return true;
116 }
117
118 bool
handleLOP2(Instruction * i)119 GV100LegalizeSSA::handleLOP2(Instruction *i)
120 {
121 uint8_t src0 = NV50_IR_SUBOP_LOP3_LUT_SRC0;
122 uint8_t src1 = NV50_IR_SUBOP_LOP3_LUT_SRC1;
123 uint8_t subOp;
124
125 if (i->src(0).mod & Modifier(NV50_IR_MOD_NOT))
126 src0 = ~src0;
127 if (i->src(1).mod & Modifier(NV50_IR_MOD_NOT))
128 src1 = ~src1;
129
130 switch (i->op) {
131 case OP_AND: subOp = src0 & src1; break;
132 case OP_OR : subOp = src0 | src1; break;
133 case OP_XOR: subOp = src0 ^ src1; break;
134 default:
135 unreachable("invalid LOP2 opcode");
136 }
137
138 bld.mkOp3(OP_LOP3_LUT, TYPE_U32, i->getDef(0), i->getSrc(0), i->getSrc(1),
139 bld.mkImm(0))->subOp = subOp;
140 return true;
141 }
142
143 bool
handleNOT(Instruction * i)144 GV100LegalizeSSA::handleNOT(Instruction *i)
145 {
146 bld.mkOp3(OP_LOP3_LUT, TYPE_U32, i->getDef(0), bld.mkImm(0), i->getSrc(0),
147 bld.mkImm(0))->subOp = (uint8_t)~NV50_IR_SUBOP_LOP3_LUT_SRC1;
148 return true;
149 }
150
151 bool
handlePREEX2(Instruction * i)152 GV100LegalizeSSA::handlePREEX2(Instruction *i)
153 {
154 i->def(0).replace(i->src(0), false);
155 return true;
156 }
157
158 bool
handleQUADON(Instruction * i)159 GV100LegalizeSSA::handleQUADON(Instruction *i)
160 {
161 handleSHFL(i); // Inserts OP_WARPSYNC
162 return true;
163 }
164
165 bool
handleQUADPOP(Instruction * i)166 GV100LegalizeSSA::handleQUADPOP(Instruction *i)
167 {
168 return true;
169 }
170
171 bool
handleSET(Instruction * i)172 GV100LegalizeSSA::handleSET(Instruction *i)
173 {
174 Value *src2 = i->srcExists(2) ? i->getSrc(2) : NULL;
175 Value *pred = bld.getSSA(1, FILE_PREDICATE), *met;
176 Instruction *xsetp;
177
178 if (isFloatType(i->dType)) {
179 if (i->sType == TYPE_F32)
180 return false; // HW has FSET.BF
181 met = bld.mkImm(0x3f800000);
182 } else {
183 met = bld.mkImm(0xffffffff);
184 }
185
186 xsetp = bld.mkCmp(i->op, i->asCmp()->setCond, TYPE_U8, pred, i->sType,
187 i->getSrc(0), i->getSrc(1));
188 xsetp->src(0).mod = i->src(0).mod;
189 xsetp->src(1).mod = i->src(1).mod;
190 xsetp->setSrc(2, src2);
191 xsetp->ftz = i->ftz;
192
193 i = bld.mkOp3(OP_SELP, TYPE_U32, i->getDef(0), bld.mkImm(0), met, pred);
194 i->src(2).mod = Modifier(NV50_IR_MOD_NOT);
195 return true;
196 }
197
198 bool
handleSHFL(Instruction * i)199 GV100LegalizeSSA::handleSHFL(Instruction *i)
200 {
201 Instruction *sync = new_Instruction(func, OP_WARPSYNC, TYPE_NONE);
202 sync->fixed = 1;
203 sync->setSrc(0, bld.mkImm(0xffffffff));
204 i->bb->insertBefore(i, sync);
205 return false;
206 }
207
208 bool
handleShift(Instruction * i)209 GV100LegalizeSSA::handleShift(Instruction *i)
210 {
211 Value *zero = bld.mkImm(0);
212 Value *src1 = i->getSrc(1);
213 Value *src0, *src2;
214 uint8_t subOp = i->op == OP_SHL ? NV50_IR_SUBOP_SHF_L : NV50_IR_SUBOP_SHF_R;
215
216 if (i->op == OP_SHL && i->src(0).getFile() == FILE_GPR) {
217 src0 = i->getSrc(0);
218 src2 = zero;
219 } else {
220 src0 = zero;
221 src2 = i->getSrc(0);
222 subOp |= NV50_IR_SUBOP_SHF_HI;
223 }
224 if (i->subOp & NV50_IR_SUBOP_SHIFT_WRAP)
225 subOp |= NV50_IR_SUBOP_SHF_W;
226
227 bld.mkOp3(OP_SHF, i->dType, i->getDef(0), src0, src1, src2)->subOp = subOp;
228 return true;
229 }
230
231 bool
handleSUB(Instruction * i)232 GV100LegalizeSSA::handleSUB(Instruction *i)
233 {
234 Instruction *xadd =
235 bld.mkOp2(OP_ADD, i->dType, i->getDef(0), i->getSrc(0), i->getSrc(1));
236 xadd->src(0).mod = i->src(0).mod;
237 xadd->src(1).mod = i->src(1).mod ^ Modifier(NV50_IR_MOD_NEG);
238 xadd->ftz = i->ftz;
239 return true;
240 }
241
242 bool
visit(Instruction * i)243 GV100LegalizeSSA::visit(Instruction *i)
244 {
245 bool lowered = false;
246
247 bld.setPosition(i, false);
248 if (i->sType == TYPE_F32 && i->dType != TYPE_F16 &&
249 prog->getType() != Program::TYPE_COMPUTE)
250 handleFTZ(i);
251
252 switch (i->op) {
253 case OP_AND:
254 case OP_OR:
255 case OP_XOR:
256 if (i->def(0).getFile() != FILE_PREDICATE)
257 lowered = handleLOP2(i);
258 break;
259 case OP_NOT:
260 lowered = handleNOT(i);
261 break;
262 case OP_SHL:
263 case OP_SHR:
264 lowered = handleShift(i);
265 break;
266 case OP_SET:
267 case OP_SET_AND:
268 case OP_SET_OR:
269 case OP_SET_XOR:
270 if (i->def(0).getFile() != FILE_PREDICATE)
271 lowered = handleSET(i);
272 break;
273 case OP_SLCT:
274 lowered = handleCMP(i);
275 break;
276 case OP_PREEX2:
277 lowered = handlePREEX2(i);
278 break;
279 case OP_MUL:
280 if (!isFloatType(i->dType))
281 lowered = handleIMUL(i);
282 break;
283 case OP_MAD:
284 if (!isFloatType(i->dType) && i->subOp == NV50_IR_SUBOP_MUL_HIGH)
285 lowered = handleIMAD_HIGH(i);
286 break;
287 case OP_SHFL:
288 lowered = handleSHFL(i);
289 break;
290 case OP_QUADON:
291 lowered = handleQUADON(i);
292 break;
293 case OP_QUADPOP:
294 lowered = handleQUADPOP(i);
295 break;
296 case OP_SUB:
297 lowered = handleSUB(i);
298 break;
299 case OP_MAX:
300 case OP_MIN:
301 if (!isFloatType(i->dType))
302 lowered = handleIMNMX(i);
303 break;
304 case OP_ADD:
305 if (!isFloatType(i->dType) && typeSizeof(i->dType) == 8)
306 lowered = handleIADD64(i);
307 break;
308 case OP_PFETCH:
309 handlePFETCH(i);
310 break;
311 case OP_LOAD:
312 handleLOAD(i);
313 break;
314 default:
315 break;
316 }
317
318 if (lowered)
319 delete_Instruction(prog, i);
320
321 return true;
322 }
323
324 bool
handleDMNMX(Instruction * i)325 GV100LoweringPass::handleDMNMX(Instruction *i)
326 {
327 Value *pred = bld.getSSA(1, FILE_PREDICATE);
328 Value *src0[2], *src1[2], *dest[2];
329
330 bld.mkCmp(OP_SET, (i->op == OP_MIN) ? CC_LT : CC_GT, TYPE_U32, pred,
331 i->sType, i->getSrc(0), i->getSrc(1));
332 bld.mkSplit(src0, 4, i->getSrc(0));
333 bld.mkSplit(src1, 4, i->getSrc(1));
334 bld.mkSplit(dest, 4, i->getDef(0));
335 bld.mkOp3(OP_SELP, TYPE_U32, dest[0], src0[0], src1[0], pred);
336 bld.mkOp3(OP_SELP, TYPE_U32, dest[1], src0[1], src1[1], pred);
337 bld.mkOp2(OP_MERGE, TYPE_U64, i->getDef(0), dest[0], dest[1]);
338 return true;
339 }
340
341 bool
handleEXTBF(Instruction * i)342 GV100LoweringPass::handleEXTBF(Instruction *i)
343 {
344 Value *bit = bld.getScratch();
345 Value *cnt = bld.getScratch();
346 Value *mask = bld.getScratch();
347 Value *zero = bld.mkImm(0);
348
349 bld.mkOp3(OP_PERMT, TYPE_U32, bit, i->getSrc(1), bld.mkImm(0x4440), zero);
350 bld.mkOp3(OP_PERMT, TYPE_U32, cnt, i->getSrc(1), bld.mkImm(0x4441), zero);
351 bld.mkOp2(OP_BMSK, TYPE_U32, mask, bit, cnt);
352 bld.mkOp2(OP_AND, TYPE_U32, mask, i->getSrc(0), mask);
353 bld.mkOp2(OP_SHR, TYPE_U32, i->getDef(0), mask, bit);
354 if (isSignedType(i->dType))
355 bld.mkOp2(OP_SGXT, TYPE_S32, i->getDef(0), i->getDef(0), cnt);
356
357 return true;
358 }
359
360 bool
handleFLOW(Instruction * i)361 GV100LoweringPass::handleFLOW(Instruction *i)
362 {
363 i->op = OP_BRA;
364 return false;
365 }
366
367 bool
handleI2I(Instruction * i)368 GV100LoweringPass::handleI2I(Instruction *i)
369 {
370 bld.mkCvt(OP_CVT, TYPE_F32, i->getDef(0), i->sType, i->getSrc(0))->
371 subOp = i->subOp;
372 bld.mkCvt(OP_CVT, i->dType, i->getDef(0), TYPE_F32, i->getDef(0));
373 return true;
374 }
375
376 bool
handleINSBF(Instruction * i)377 GV100LoweringPass::handleINSBF(Instruction *i)
378 {
379 Value *bit = bld.getScratch();
380 Value *cnt = bld.getScratch();
381 Value *mask = bld.getScratch();
382 Value *src0 = bld.getScratch();
383 Value *zero = bld.mkImm(0);
384
385 bld.mkOp3(OP_PERMT, TYPE_U32, bit, i->getSrc(1), bld.mkImm(0x4440), zero);
386 bld.mkOp3(OP_PERMT, TYPE_U32, cnt, i->getSrc(1), bld.mkImm(0x4441), zero);
387 bld.mkOp2(OP_BMSK, TYPE_U32, mask, zero, cnt);
388
389 bld.mkOp2(OP_AND, TYPE_U32, src0, i->getSrc(0), mask);
390 bld.mkOp2(OP_SHL, TYPE_U32, src0, src0, bit);
391
392 bld.mkOp2(OP_SHL, TYPE_U32, mask, mask, bit);
393 bld.mkOp3(OP_LOP3_LUT, TYPE_U32, i->getDef(0), src0, i->getSrc(2), mask)->
394 subOp = NV50_IR_SUBOP_LOP3_LUT(a | (b & ~c));
395
396 return true;
397 }
398
399 bool
handlePINTERP(Instruction * i)400 GV100LoweringPass::handlePINTERP(Instruction *i)
401 {
402 Value *src2 = i->srcExists(2) ? i->getSrc(2) : NULL;
403 Instruction *ipa, *mul;
404
405 ipa = bld.mkOp2(OP_LINTERP, TYPE_F32, i->getDef(0), i->getSrc(0), src2);
406 ipa->ipa = i->ipa;
407 mul = bld.mkOp2(OP_MUL, TYPE_F32, i->getDef(0), i->getDef(0), i->getSrc(1));
408
409 if (i->getInterpMode() == NV50_IR_INTERP_SC) {
410 ipa->setDef(1, bld.getSSA(1, FILE_PREDICATE));
411 mul->setPredicate(CC_NOT_P, ipa->getDef(1));
412 }
413
414 return true;
415 }
416
417 bool
handlePREFLOW(Instruction * i)418 GV100LoweringPass::handlePREFLOW(Instruction *i)
419 {
420 return true;
421 }
422
423 bool
handlePRESIN(Instruction * i)424 GV100LoweringPass::handlePRESIN(Instruction *i)
425 {
426 const float f = 1.0 / (2.0 * 3.14159265);
427 bld.mkOp2(OP_MUL, i->dType, i->getDef(0), i->getSrc(0), bld.mkImm(f));
428 return true;
429 }
430
431 bool
visit(Instruction * i)432 GV100LoweringPass::visit(Instruction *i)
433 {
434 bool lowered = false;
435
436 bld.setPosition(i, false);
437
438 switch (i->op) {
439 case OP_BREAK:
440 case OP_CONT:
441 lowered = handleFLOW(i);
442 break;
443 case OP_PREBREAK:
444 case OP_PRECONT:
445 lowered = handlePREFLOW(i);
446 break;
447 case OP_CVT:
448 if (i->src(0).getFile() != FILE_PREDICATE &&
449 i->def(0).getFile() != FILE_PREDICATE &&
450 !isFloatType(i->dType) && !isFloatType(i->sType))
451 lowered = handleI2I(i);
452 break;
453 case OP_EXTBF:
454 lowered = handleEXTBF(i);
455 break;
456 case OP_INSBF:
457 lowered = handleINSBF(i);
458 break;
459 case OP_MAX:
460 case OP_MIN:
461 if (i->dType == TYPE_F64)
462 lowered = handleDMNMX(i);
463 break;
464 case OP_PINTERP:
465 lowered = handlePINTERP(i);
466 break;
467 case OP_PRESIN:
468 lowered = handlePRESIN(i);
469 break;
470 default:
471 break;
472 }
473
474 if (lowered)
475 delete_Instruction(prog, i);
476
477 return true;
478 }
479
480 } // namespace nv50_ir
481