1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /** @file brw_fs_visitor.cpp
25 *
26 * This file supports generating the FS LIR from the GLSL IR. The LIR
27 * makes it easier to do backend-specific optimizations than doing so
28 * in the GLSL IR or in the native code.
29 */
30 #include "brw_fs.h"
31 #include "compiler/glsl_types.h"
32
33 using namespace brw;
34
35 /* Sample from the MCS surface attached to this multisample texture. */
36 fs_reg
emit_mcs_fetch(const fs_reg & coordinate,unsigned components,const fs_reg & texture,const fs_reg & texture_handle)37 fs_visitor::emit_mcs_fetch(const fs_reg &coordinate, unsigned components,
38 const fs_reg &texture,
39 const fs_reg &texture_handle)
40 {
41 const fs_reg dest = vgrf(glsl_type::uvec4_type);
42
43 fs_reg srcs[TEX_LOGICAL_NUM_SRCS];
44 srcs[TEX_LOGICAL_SRC_COORDINATE] = coordinate;
45 srcs[TEX_LOGICAL_SRC_SURFACE] = texture;
46 srcs[TEX_LOGICAL_SRC_SAMPLER] = brw_imm_ud(0);
47 srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE] = texture_handle;
48 srcs[TEX_LOGICAL_SRC_COORD_COMPONENTS] = brw_imm_d(components);
49 srcs[TEX_LOGICAL_SRC_GRAD_COMPONENTS] = brw_imm_d(0);
50
51 fs_inst *inst = bld.emit(SHADER_OPCODE_TXF_MCS_LOGICAL, dest, srcs,
52 ARRAY_SIZE(srcs));
53
54 /* We only care about one or two regs of response, but the sampler always
55 * writes 4/8.
56 */
57 inst->size_written = 4 * dest.component_size(inst->exec_size);
58
59 return dest;
60 }
61
62 /**
63 * Apply workarounds for Gen6 gather with UINT/SINT
64 */
65 void
emit_gen6_gather_wa(uint8_t wa,fs_reg dst)66 fs_visitor::emit_gen6_gather_wa(uint8_t wa, fs_reg dst)
67 {
68 if (!wa)
69 return;
70
71 int width = (wa & WA_8BIT) ? 8 : 16;
72
73 for (int i = 0; i < 4; i++) {
74 fs_reg dst_f = retype(dst, BRW_REGISTER_TYPE_F);
75 /* Convert from UNORM to UINT */
76 bld.MUL(dst_f, dst_f, brw_imm_f((1 << width) - 1));
77 bld.MOV(dst, dst_f);
78
79 if (wa & WA_SIGN) {
80 /* Reinterpret the UINT value as a signed INT value by
81 * shifting the sign bit into place, then shifting back
82 * preserving sign.
83 */
84 bld.SHL(dst, dst, brw_imm_d(32 - width));
85 bld.ASR(dst, dst, brw_imm_d(32 - width));
86 }
87
88 dst = offset(dst, bld, 1);
89 }
90 }
91
92 /** Emits a dummy fragment shader consisting of magenta for bringup purposes. */
93 void
emit_dummy_fs()94 fs_visitor::emit_dummy_fs()
95 {
96 int reg_width = dispatch_width / 8;
97
98 /* Everyone's favorite color. */
99 const float color[4] = { 1.0, 0.0, 1.0, 0.0 };
100 for (int i = 0; i < 4; i++) {
101 bld.MOV(fs_reg(MRF, 2 + i * reg_width, BRW_REGISTER_TYPE_F),
102 brw_imm_f(color[i]));
103 }
104
105 fs_inst *write;
106 write = bld.emit(FS_OPCODE_FB_WRITE);
107 write->eot = true;
108 write->last_rt = true;
109 if (devinfo->gen >= 6) {
110 write->base_mrf = 2;
111 write->mlen = 4 * reg_width;
112 } else {
113 write->header_size = 2;
114 write->base_mrf = 0;
115 write->mlen = 2 + 4 * reg_width;
116 }
117
118 /* Tell the SF we don't have any inputs. Gen4-5 require at least one
119 * varying to avoid GPU hangs, so set that.
120 */
121 struct brw_wm_prog_data *wm_prog_data = brw_wm_prog_data(this->prog_data);
122 wm_prog_data->num_varying_inputs = devinfo->gen < 6 ? 1 : 0;
123 memset(wm_prog_data->urb_setup, -1,
124 sizeof(wm_prog_data->urb_setup[0]) * VARYING_SLOT_MAX);
125 brw_compute_urb_setup_index(wm_prog_data);
126
127 /* We don't have any uniforms. */
128 stage_prog_data->nr_params = 0;
129 stage_prog_data->nr_pull_params = 0;
130 stage_prog_data->curb_read_length = 0;
131 stage_prog_data->dispatch_grf_start_reg = 2;
132 wm_prog_data->dispatch_grf_start_reg_16 = 2;
133 wm_prog_data->dispatch_grf_start_reg_32 = 2;
134 grf_used = 1; /* Gen4-5 don't allow zero GRF blocks */
135
136 calculate_cfg();
137 }
138
139 /* The register location here is relative to the start of the URB
140 * data. It will get adjusted to be a real location before
141 * generate_code() time.
142 */
143 fs_reg
interp_reg(int location,int channel)144 fs_visitor::interp_reg(int location, int channel)
145 {
146 assert(stage == MESA_SHADER_FRAGMENT);
147 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
148 int regnr = prog_data->urb_setup[location] * 4 + channel;
149 assert(prog_data->urb_setup[location] != -1);
150
151 return fs_reg(ATTR, regnr, BRW_REGISTER_TYPE_F);
152 }
153
154 /** Emits the interpolation for the varying inputs. */
155 void
emit_interpolation_setup_gen4()156 fs_visitor::emit_interpolation_setup_gen4()
157 {
158 struct brw_reg g1_uw = retype(brw_vec1_grf(1, 0), BRW_REGISTER_TYPE_UW);
159
160 fs_builder abld = bld.annotate("compute pixel centers");
161 this->pixel_x = vgrf(glsl_type::uint_type);
162 this->pixel_y = vgrf(glsl_type::uint_type);
163 this->pixel_x.type = BRW_REGISTER_TYPE_UW;
164 this->pixel_y.type = BRW_REGISTER_TYPE_UW;
165 abld.ADD(this->pixel_x,
166 fs_reg(stride(suboffset(g1_uw, 4), 2, 4, 0)),
167 fs_reg(brw_imm_v(0x10101010)));
168 abld.ADD(this->pixel_y,
169 fs_reg(stride(suboffset(g1_uw, 5), 2, 4, 0)),
170 fs_reg(brw_imm_v(0x11001100)));
171
172 abld = bld.annotate("compute pixel deltas from v0");
173
174 this->delta_xy[BRW_BARYCENTRIC_PERSPECTIVE_PIXEL] =
175 vgrf(glsl_type::vec2_type);
176 const fs_reg &delta_xy = this->delta_xy[BRW_BARYCENTRIC_PERSPECTIVE_PIXEL];
177 const fs_reg xstart(negate(brw_vec1_grf(1, 0)));
178 const fs_reg ystart(negate(brw_vec1_grf(1, 1)));
179
180 if (devinfo->has_pln) {
181 for (unsigned i = 0; i < dispatch_width / 8; i++) {
182 abld.quarter(i).ADD(quarter(offset(delta_xy, abld, 0), i),
183 quarter(this->pixel_x, i), xstart);
184 abld.quarter(i).ADD(quarter(offset(delta_xy, abld, 1), i),
185 quarter(this->pixel_y, i), ystart);
186 }
187 } else {
188 abld.ADD(offset(delta_xy, abld, 0), this->pixel_x, xstart);
189 abld.ADD(offset(delta_xy, abld, 1), this->pixel_y, ystart);
190 }
191
192 abld = bld.annotate("compute pos.w and 1/pos.w");
193 /* Compute wpos.w. It's always in our setup, since it's needed to
194 * interpolate the other attributes.
195 */
196 this->wpos_w = vgrf(glsl_type::float_type);
197 abld.emit(FS_OPCODE_LINTERP, wpos_w, delta_xy,
198 component(interp_reg(VARYING_SLOT_POS, 3), 0));
199 /* Compute the pixel 1/W value from wpos.w. */
200 this->pixel_w = vgrf(glsl_type::float_type);
201 abld.emit(SHADER_OPCODE_RCP, this->pixel_w, wpos_w);
202 }
203
204 static unsigned
brw_rnd_mode_from_nir(unsigned mode,unsigned * mask)205 brw_rnd_mode_from_nir(unsigned mode, unsigned *mask)
206 {
207 unsigned brw_mode = 0;
208 *mask = 0;
209
210 if ((FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 |
211 FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 |
212 FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64) &
213 mode) {
214 brw_mode |= BRW_RND_MODE_RTZ << BRW_CR0_RND_MODE_SHIFT;
215 *mask |= BRW_CR0_RND_MODE_MASK;
216 }
217 if ((FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 |
218 FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32 |
219 FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64) &
220 mode) {
221 brw_mode |= BRW_RND_MODE_RTNE << BRW_CR0_RND_MODE_SHIFT;
222 *mask |= BRW_CR0_RND_MODE_MASK;
223 }
224 if (mode & FLOAT_CONTROLS_DENORM_PRESERVE_FP16) {
225 brw_mode |= BRW_CR0_FP16_DENORM_PRESERVE;
226 *mask |= BRW_CR0_FP16_DENORM_PRESERVE;
227 }
228 if (mode & FLOAT_CONTROLS_DENORM_PRESERVE_FP32) {
229 brw_mode |= BRW_CR0_FP32_DENORM_PRESERVE;
230 *mask |= BRW_CR0_FP32_DENORM_PRESERVE;
231 }
232 if (mode & FLOAT_CONTROLS_DENORM_PRESERVE_FP64) {
233 brw_mode |= BRW_CR0_FP64_DENORM_PRESERVE;
234 *mask |= BRW_CR0_FP64_DENORM_PRESERVE;
235 }
236 if (mode & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16)
237 *mask |= BRW_CR0_FP16_DENORM_PRESERVE;
238 if (mode & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32)
239 *mask |= BRW_CR0_FP32_DENORM_PRESERVE;
240 if (mode & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64)
241 *mask |= BRW_CR0_FP64_DENORM_PRESERVE;
242 if (mode == FLOAT_CONTROLS_DEFAULT_FLOAT_CONTROL_MODE)
243 *mask |= BRW_CR0_FP_MODE_MASK;
244
245 if (*mask != 0)
246 assert((*mask & brw_mode) == brw_mode);
247
248 return brw_mode;
249 }
250
251 void
emit_shader_float_controls_execution_mode()252 fs_visitor::emit_shader_float_controls_execution_mode()
253 {
254 unsigned execution_mode = this->nir->info.float_controls_execution_mode;
255 if (execution_mode == FLOAT_CONTROLS_DEFAULT_FLOAT_CONTROL_MODE)
256 return;
257
258 fs_builder abld = bld.annotate("shader floats control execution mode");
259 unsigned mask, mode = brw_rnd_mode_from_nir(execution_mode, &mask);
260
261 if (mask == 0)
262 return;
263
264 abld.emit(SHADER_OPCODE_FLOAT_CONTROL_MODE, bld.null_reg_ud(),
265 brw_imm_d(mode), brw_imm_d(mask));
266 }
267
268 /** Emits the interpolation for the varying inputs. */
269 void
emit_interpolation_setup_gen6()270 fs_visitor::emit_interpolation_setup_gen6()
271 {
272 fs_builder abld = bld.annotate("compute pixel centers");
273
274 this->pixel_x = vgrf(glsl_type::float_type);
275 this->pixel_y = vgrf(glsl_type::float_type);
276
277 for (unsigned i = 0; i < DIV_ROUND_UP(dispatch_width, 16); i++) {
278 const fs_builder hbld = abld.group(MIN2(16, dispatch_width), i);
279 struct brw_reg gi_uw = retype(brw_vec1_grf(1 + i, 0), BRW_REGISTER_TYPE_UW);
280
281 if (devinfo->gen >= 8 || dispatch_width == 8) {
282 /* The "Register Region Restrictions" page says for BDW (and newer,
283 * presumably):
284 *
285 * "When destination spans two registers, the source may be one or
286 * two registers. The destination elements must be evenly split
287 * between the two registers."
288 *
289 * Thus we can do a single add(16) in SIMD8 or an add(32) in SIMD16
290 * to compute our pixel centers.
291 */
292 const fs_builder dbld =
293 abld.exec_all().group(hbld.dispatch_width() * 2, 0);
294 fs_reg int_pixel_xy = dbld.vgrf(BRW_REGISTER_TYPE_UW);
295
296 dbld.ADD(int_pixel_xy,
297 fs_reg(stride(suboffset(gi_uw, 4), 1, 4, 0)),
298 fs_reg(brw_imm_v(0x11001010)));
299
300 hbld.emit(FS_OPCODE_PIXEL_X, offset(pixel_x, hbld, i), int_pixel_xy);
301 hbld.emit(FS_OPCODE_PIXEL_Y, offset(pixel_y, hbld, i), int_pixel_xy);
302 } else {
303 /* The "Register Region Restrictions" page says for SNB, IVB, HSW:
304 *
305 * "When destination spans two registers, the source MUST span
306 * two registers."
307 *
308 * Since the GRF source of the ADD will only read a single register,
309 * we must do two separate ADDs in SIMD16.
310 */
311 const fs_reg int_pixel_x = hbld.vgrf(BRW_REGISTER_TYPE_UW);
312 const fs_reg int_pixel_y = hbld.vgrf(BRW_REGISTER_TYPE_UW);
313
314 hbld.ADD(int_pixel_x,
315 fs_reg(stride(suboffset(gi_uw, 4), 2, 4, 0)),
316 fs_reg(brw_imm_v(0x10101010)));
317 hbld.ADD(int_pixel_y,
318 fs_reg(stride(suboffset(gi_uw, 5), 2, 4, 0)),
319 fs_reg(brw_imm_v(0x11001100)));
320
321 /* As of gen6, we can no longer mix float and int sources. We have
322 * to turn the integer pixel centers into floats for their actual
323 * use.
324 */
325 hbld.MOV(offset(pixel_x, hbld, i), int_pixel_x);
326 hbld.MOV(offset(pixel_y, hbld, i), int_pixel_y);
327 }
328 }
329
330 abld = bld.annotate("compute pos.w");
331 this->pixel_w = fetch_payload_reg(abld, payload.source_w_reg);
332 this->wpos_w = vgrf(glsl_type::float_type);
333 abld.emit(SHADER_OPCODE_RCP, this->wpos_w, this->pixel_w);
334
335 struct brw_wm_prog_data *wm_prog_data = brw_wm_prog_data(prog_data);
336
337 for (int i = 0; i < BRW_BARYCENTRIC_MODE_COUNT; ++i) {
338 this->delta_xy[i] = fetch_barycentric_reg(
339 bld, payload.barycentric_coord_reg[i]);
340 }
341
342 uint32_t centroid_modes = wm_prog_data->barycentric_interp_modes &
343 (1 << BRW_BARYCENTRIC_PERSPECTIVE_CENTROID |
344 1 << BRW_BARYCENTRIC_NONPERSPECTIVE_CENTROID);
345
346 if (devinfo->needs_unlit_centroid_workaround && centroid_modes) {
347 /* Get the pixel/sample mask into f0 so that we know which
348 * pixels are lit. Then, for each channel that is unlit,
349 * replace the centroid data with non-centroid data.
350 */
351 for (unsigned i = 0; i < DIV_ROUND_UP(dispatch_width, 16); i++) {
352 bld.exec_all().group(1, 0)
353 .MOV(retype(brw_flag_reg(0, i), BRW_REGISTER_TYPE_UW),
354 retype(brw_vec1_grf(1 + i, 7), BRW_REGISTER_TYPE_UW));
355 }
356
357 for (int i = 0; i < BRW_BARYCENTRIC_MODE_COUNT; ++i) {
358 if (!(centroid_modes & (1 << i)))
359 continue;
360
361 const fs_reg centroid_delta_xy = delta_xy[i];
362 const fs_reg &pixel_delta_xy = delta_xy[i - 1];
363
364 delta_xy[i] = bld.vgrf(BRW_REGISTER_TYPE_F, 2);
365
366 for (unsigned c = 0; c < 2; c++) {
367 for (unsigned q = 0; q < dispatch_width / 8; q++) {
368 set_predicate(BRW_PREDICATE_NORMAL,
369 bld.quarter(q).SEL(
370 quarter(offset(delta_xy[i], bld, c), q),
371 quarter(offset(centroid_delta_xy, bld, c), q),
372 quarter(offset(pixel_delta_xy, bld, c), q)));
373 }
374 }
375 }
376 }
377 }
378
379 static enum brw_conditional_mod
cond_for_alpha_func(GLenum func)380 cond_for_alpha_func(GLenum func)
381 {
382 switch(func) {
383 case GL_GREATER:
384 return BRW_CONDITIONAL_G;
385 case GL_GEQUAL:
386 return BRW_CONDITIONAL_GE;
387 case GL_LESS:
388 return BRW_CONDITIONAL_L;
389 case GL_LEQUAL:
390 return BRW_CONDITIONAL_LE;
391 case GL_EQUAL:
392 return BRW_CONDITIONAL_EQ;
393 case GL_NOTEQUAL:
394 return BRW_CONDITIONAL_NEQ;
395 default:
396 unreachable("Not reached");
397 }
398 }
399
400 /**
401 * Alpha test support for when we compile it into the shader instead
402 * of using the normal fixed-function alpha test.
403 */
404 void
emit_alpha_test()405 fs_visitor::emit_alpha_test()
406 {
407 assert(stage == MESA_SHADER_FRAGMENT);
408 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
409 const fs_builder abld = bld.annotate("Alpha test");
410
411 fs_inst *cmp;
412 if (key->alpha_test_func == GL_ALWAYS)
413 return;
414
415 if (key->alpha_test_func == GL_NEVER) {
416 /* f0.1 = 0 */
417 fs_reg some_reg = fs_reg(retype(brw_vec8_grf(0, 0),
418 BRW_REGISTER_TYPE_UW));
419 cmp = abld.CMP(bld.null_reg_f(), some_reg, some_reg,
420 BRW_CONDITIONAL_NEQ);
421 } else {
422 /* RT0 alpha */
423 fs_reg color = offset(outputs[0], bld, 3);
424
425 /* f0.1 &= func(color, ref) */
426 cmp = abld.CMP(bld.null_reg_f(), color, brw_imm_f(key->alpha_test_ref),
427 cond_for_alpha_func(key->alpha_test_func));
428 }
429 cmp->predicate = BRW_PREDICATE_NORMAL;
430 cmp->flag_subreg = 1;
431 }
432
433 fs_inst *
emit_single_fb_write(const fs_builder & bld,fs_reg color0,fs_reg color1,fs_reg src0_alpha,unsigned components)434 fs_visitor::emit_single_fb_write(const fs_builder &bld,
435 fs_reg color0, fs_reg color1,
436 fs_reg src0_alpha, unsigned components)
437 {
438 assert(stage == MESA_SHADER_FRAGMENT);
439 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
440
441 /* Hand over gl_FragDepth or the payload depth. */
442 const fs_reg dst_depth = fetch_payload_reg(bld, payload.dest_depth_reg);
443 fs_reg src_depth, src_stencil;
444
445 if (source_depth_to_render_target) {
446 if (nir->info.outputs_written & BITFIELD64_BIT(FRAG_RESULT_DEPTH))
447 src_depth = frag_depth;
448 else
449 src_depth = fetch_payload_reg(bld, payload.source_depth_reg);
450 }
451
452 if (nir->info.outputs_written & BITFIELD64_BIT(FRAG_RESULT_STENCIL))
453 src_stencil = frag_stencil;
454
455 const fs_reg sources[] = {
456 color0, color1, src0_alpha, src_depth, dst_depth, src_stencil,
457 (prog_data->uses_omask ? sample_mask : fs_reg()),
458 brw_imm_ud(components)
459 };
460 assert(ARRAY_SIZE(sources) - 1 == FB_WRITE_LOGICAL_SRC_COMPONENTS);
461 fs_inst *write = bld.emit(FS_OPCODE_FB_WRITE_LOGICAL, fs_reg(),
462 sources, ARRAY_SIZE(sources));
463
464 if (prog_data->uses_kill) {
465 write->predicate = BRW_PREDICATE_NORMAL;
466 write->flag_subreg = sample_mask_flag_subreg(this);
467 }
468
469 return write;
470 }
471
472 void
emit_fb_writes()473 fs_visitor::emit_fb_writes()
474 {
475 assert(stage == MESA_SHADER_FRAGMENT);
476 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
477 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
478
479 fs_inst *inst = NULL;
480
481 if (source_depth_to_render_target && devinfo->gen == 6) {
482 /* For outputting oDepth on gen6, SIMD8 writes have to be used. This
483 * would require SIMD8 moves of each half to message regs, e.g. by using
484 * the SIMD lowering pass. Unfortunately this is more difficult than it
485 * sounds because the SIMD8 single-source message lacks channel selects
486 * for the second and third subspans.
487 */
488 limit_dispatch_width(8, "Depth writes unsupported in SIMD16+ mode.\n");
489 }
490
491 if (nir->info.outputs_written & BITFIELD64_BIT(FRAG_RESULT_STENCIL)) {
492 /* From the 'Render Target Write message' section of the docs:
493 * "Output Stencil is not supported with SIMD16 Render Target Write
494 * Messages."
495 */
496 limit_dispatch_width(8, "gl_FragStencilRefARB unsupported "
497 "in SIMD16+ mode.\n");
498 }
499
500 /* ANV doesn't know about sample mask output during the wm key creation
501 * so we compute if we need replicate alpha and emit alpha to coverage
502 * workaround here.
503 */
504 const bool replicate_alpha = key->alpha_test_replicate_alpha ||
505 (key->nr_color_regions > 1 && key->alpha_to_coverage &&
506 (sample_mask.file == BAD_FILE || devinfo->gen == 6));
507
508 for (int target = 0; target < key->nr_color_regions; target++) {
509 /* Skip over outputs that weren't written. */
510 if (this->outputs[target].file == BAD_FILE)
511 continue;
512
513 const fs_builder abld = bld.annotate(
514 ralloc_asprintf(this->mem_ctx, "FB write target %d", target));
515
516 fs_reg src0_alpha;
517 if (devinfo->gen >= 6 && replicate_alpha && target != 0)
518 src0_alpha = offset(outputs[0], bld, 3);
519
520 inst = emit_single_fb_write(abld, this->outputs[target],
521 this->dual_src_output, src0_alpha, 4);
522 inst->target = target;
523 }
524
525 prog_data->dual_src_blend = (this->dual_src_output.file != BAD_FILE &&
526 this->outputs[0].file != BAD_FILE);
527 assert(!prog_data->dual_src_blend || key->nr_color_regions == 1);
528
529 if (inst == NULL) {
530 /* Even if there's no color buffers enabled, we still need to send
531 * alpha out the pipeline to our null renderbuffer to support
532 * alpha-testing, alpha-to-coverage, and so on.
533 */
534 /* FINISHME: Factor out this frequently recurring pattern into a
535 * helper function.
536 */
537 const fs_reg srcs[] = { reg_undef, reg_undef,
538 reg_undef, offset(this->outputs[0], bld, 3) };
539 const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD, 4);
540 bld.LOAD_PAYLOAD(tmp, srcs, 4, 0);
541
542 inst = emit_single_fb_write(bld, tmp, reg_undef, reg_undef, 4);
543 inst->target = 0;
544 }
545
546 inst->last_rt = true;
547 inst->eot = true;
548
549 if (devinfo->gen >= 11 && devinfo->gen <= 12 &&
550 prog_data->dual_src_blend) {
551 /* The dual-source RT write messages fail to release the thread
552 * dependency on ICL and TGL with SIMD32 dispatch, leading to hangs.
553 *
554 * XXX - Emit an extra single-source NULL RT-write marked LastRT in
555 * order to release the thread dependency without disabling
556 * SIMD32.
557 *
558 * The dual-source RT write messages may lead to hangs with SIMD16
559 * dispatch on ICL due some unknown reasons, see
560 * https://gitlab.freedesktop.org/mesa/mesa/-/issues/2183
561 */
562 limit_dispatch_width(8, "Dual source blending unsupported "
563 "in SIMD16 and SIMD32 modes.\n");
564 }
565 }
566
567 void
emit_urb_writes(const fs_reg & gs_vertex_count)568 fs_visitor::emit_urb_writes(const fs_reg &gs_vertex_count)
569 {
570 int slot, urb_offset, length;
571 int starting_urb_offset = 0;
572 const struct brw_vue_prog_data *vue_prog_data =
573 brw_vue_prog_data(this->prog_data);
574 const struct brw_vs_prog_key *vs_key =
575 (const struct brw_vs_prog_key *) this->key;
576 const GLbitfield64 psiz_mask =
577 VARYING_BIT_LAYER | VARYING_BIT_VIEWPORT | VARYING_BIT_PSIZ;
578 const struct brw_vue_map *vue_map = &vue_prog_data->vue_map;
579 bool flush;
580 fs_reg sources[8];
581 fs_reg urb_handle;
582
583 if (stage == MESA_SHADER_TESS_EVAL)
584 urb_handle = fs_reg(retype(brw_vec8_grf(4, 0), BRW_REGISTER_TYPE_UD));
585 else
586 urb_handle = fs_reg(retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD));
587
588 opcode opcode = SHADER_OPCODE_URB_WRITE_SIMD8;
589 int header_size = 1;
590 fs_reg per_slot_offsets;
591
592 if (stage == MESA_SHADER_GEOMETRY) {
593 const struct brw_gs_prog_data *gs_prog_data =
594 brw_gs_prog_data(this->prog_data);
595
596 /* We need to increment the Global Offset to skip over the control data
597 * header and the extra "Vertex Count" field (1 HWord) at the beginning
598 * of the VUE. We're counting in OWords, so the units are doubled.
599 */
600 starting_urb_offset = 2 * gs_prog_data->control_data_header_size_hwords;
601 if (gs_prog_data->static_vertex_count == -1)
602 starting_urb_offset += 2;
603
604 /* We also need to use per-slot offsets. The per-slot offset is the
605 * Vertex Count. SIMD8 mode processes 8 different primitives at a
606 * time; each may output a different number of vertices.
607 */
608 opcode = SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT;
609 header_size++;
610
611 /* The URB offset is in 128-bit units, so we need to multiply by 2 */
612 const int output_vertex_size_owords =
613 gs_prog_data->output_vertex_size_hwords * 2;
614
615 if (gs_vertex_count.file == IMM) {
616 per_slot_offsets = brw_imm_ud(output_vertex_size_owords *
617 gs_vertex_count.ud);
618 } else {
619 per_slot_offsets = vgrf(glsl_type::uint_type);
620 bld.MUL(per_slot_offsets, gs_vertex_count,
621 brw_imm_ud(output_vertex_size_owords));
622 }
623 }
624
625 length = 0;
626 urb_offset = starting_urb_offset;
627 flush = false;
628
629 /* SSO shaders can have VUE slots allocated which are never actually
630 * written to, so ignore them when looking for the last (written) slot.
631 */
632 int last_slot = vue_map->num_slots - 1;
633 while (last_slot > 0 &&
634 (vue_map->slot_to_varying[last_slot] == BRW_VARYING_SLOT_PAD ||
635 outputs[vue_map->slot_to_varying[last_slot]].file == BAD_FILE)) {
636 last_slot--;
637 }
638
639 bool urb_written = false;
640 for (slot = 0; slot < vue_map->num_slots; slot++) {
641 int varying = vue_map->slot_to_varying[slot];
642 switch (varying) {
643 case VARYING_SLOT_PSIZ: {
644 /* The point size varying slot is the vue header and is always in the
645 * vue map. But often none of the special varyings that live there
646 * are written and in that case we can skip writing to the vue
647 * header, provided the corresponding state properly clamps the
648 * values further down the pipeline. */
649 if ((vue_map->slots_valid & psiz_mask) == 0) {
650 assert(length == 0);
651 urb_offset++;
652 break;
653 }
654
655 fs_reg zero(VGRF, alloc.allocate(1), BRW_REGISTER_TYPE_UD);
656 bld.MOV(zero, brw_imm_ud(0u));
657
658 sources[length++] = zero;
659 if (vue_map->slots_valid & VARYING_BIT_LAYER)
660 sources[length++] = this->outputs[VARYING_SLOT_LAYER];
661 else
662 sources[length++] = zero;
663
664 if (vue_map->slots_valid & VARYING_BIT_VIEWPORT)
665 sources[length++] = this->outputs[VARYING_SLOT_VIEWPORT];
666 else
667 sources[length++] = zero;
668
669 if (vue_map->slots_valid & VARYING_BIT_PSIZ)
670 sources[length++] = this->outputs[VARYING_SLOT_PSIZ];
671 else
672 sources[length++] = zero;
673 break;
674 }
675 case BRW_VARYING_SLOT_NDC:
676 case VARYING_SLOT_EDGE:
677 unreachable("unexpected scalar vs output");
678 break;
679
680 default:
681 /* gl_Position is always in the vue map, but isn't always written by
682 * the shader. Other varyings (clip distances) get added to the vue
683 * map but don't always get written. In those cases, the
684 * corresponding this->output[] slot will be invalid we and can skip
685 * the urb write for the varying. If we've already queued up a vue
686 * slot for writing we flush a mlen 5 urb write, otherwise we just
687 * advance the urb_offset.
688 */
689 if (varying == BRW_VARYING_SLOT_PAD ||
690 this->outputs[varying].file == BAD_FILE) {
691 if (length > 0)
692 flush = true;
693 else
694 urb_offset++;
695 break;
696 }
697
698 if (stage == MESA_SHADER_VERTEX && vs_key->clamp_vertex_color &&
699 (varying == VARYING_SLOT_COL0 ||
700 varying == VARYING_SLOT_COL1 ||
701 varying == VARYING_SLOT_BFC0 ||
702 varying == VARYING_SLOT_BFC1)) {
703 /* We need to clamp these guys, so do a saturating MOV into a
704 * temp register and use that for the payload.
705 */
706 for (int i = 0; i < 4; i++) {
707 fs_reg reg = fs_reg(VGRF, alloc.allocate(1), outputs[varying].type);
708 fs_reg src = offset(this->outputs[varying], bld, i);
709 set_saturate(true, bld.MOV(reg, src));
710 sources[length++] = reg;
711 }
712 } else {
713 int slot_offset = 0;
714
715 /* When using Primitive Replication, there may be multiple slots
716 * assigned to POS.
717 */
718 if (varying == VARYING_SLOT_POS)
719 slot_offset = slot - vue_map->varying_to_slot[VARYING_SLOT_POS];
720
721 for (unsigned i = 0; i < 4; i++) {
722 sources[length++] = offset(this->outputs[varying], bld,
723 i + (slot_offset * 4));
724 }
725 }
726 break;
727 }
728
729 const fs_builder abld = bld.annotate("URB write");
730
731 /* If we've queued up 8 registers of payload (2 VUE slots), if this is
732 * the last slot or if we need to flush (see BAD_FILE varying case
733 * above), emit a URB write send now to flush out the data.
734 */
735 if (length == 8 || (length > 0 && slot == last_slot))
736 flush = true;
737 if (flush) {
738 fs_reg *payload_sources =
739 ralloc_array(mem_ctx, fs_reg, length + header_size);
740 fs_reg payload = fs_reg(VGRF, alloc.allocate(length + header_size),
741 BRW_REGISTER_TYPE_F);
742 payload_sources[0] = urb_handle;
743
744 if (opcode == SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT)
745 payload_sources[1] = per_slot_offsets;
746
747 memcpy(&payload_sources[header_size], sources,
748 length * sizeof sources[0]);
749
750 abld.LOAD_PAYLOAD(payload, payload_sources, length + header_size,
751 header_size);
752
753 fs_inst *inst = abld.emit(opcode, reg_undef, payload);
754
755 /* For ICL WA 1805992985 one needs additional write in the end. */
756 if (devinfo->gen == 11 && stage == MESA_SHADER_TESS_EVAL)
757 inst->eot = false;
758 else
759 inst->eot = slot == last_slot && stage != MESA_SHADER_GEOMETRY;
760
761 inst->mlen = length + header_size;
762 inst->offset = urb_offset;
763 urb_offset = starting_urb_offset + slot + 1;
764 length = 0;
765 flush = false;
766 urb_written = true;
767 }
768 }
769
770 /* If we don't have any valid slots to write, just do a minimal urb write
771 * send to terminate the shader. This includes 1 slot of undefined data,
772 * because it's invalid to write 0 data:
773 *
774 * From the Broadwell PRM, Volume 7: 3D Media GPGPU, Shared Functions -
775 * Unified Return Buffer (URB) > URB_SIMD8_Write and URB_SIMD8_Read >
776 * Write Data Payload:
777 *
778 * "The write data payload can be between 1 and 8 message phases long."
779 */
780 if (!urb_written) {
781 /* For GS, just turn EmitVertex() into a no-op. We don't want it to
782 * end the thread, and emit_gs_thread_end() already emits a SEND with
783 * EOT at the end of the program for us.
784 */
785 if (stage == MESA_SHADER_GEOMETRY)
786 return;
787
788 fs_reg payload = fs_reg(VGRF, alloc.allocate(2), BRW_REGISTER_TYPE_UD);
789 bld.exec_all().MOV(payload, urb_handle);
790
791 fs_inst *inst = bld.emit(SHADER_OPCODE_URB_WRITE_SIMD8, reg_undef, payload);
792 inst->eot = true;
793 inst->mlen = 2;
794 inst->offset = 1;
795 return;
796 }
797
798 /* ICL WA 1805992985:
799 *
800 * ICLLP GPU hangs on one of tessellation vkcts tests with DS not done. The
801 * send cycle, which is a urb write with an eot must be 4 phases long and
802 * all 8 lanes must valid.
803 */
804 if (devinfo->gen == 11 && stage == MESA_SHADER_TESS_EVAL) {
805 fs_reg payload = fs_reg(VGRF, alloc.allocate(6), BRW_REGISTER_TYPE_UD);
806
807 /* Workaround requires all 8 channels (lanes) to be valid. This is
808 * understood to mean they all need to be alive. First trick is to find
809 * a live channel and copy its urb handle for all the other channels to
810 * make sure all handles are valid.
811 */
812 bld.exec_all().MOV(payload, bld.emit_uniformize(urb_handle));
813
814 /* Second trick is to use masked URB write where one can tell the HW to
815 * actually write data only for selected channels even though all are
816 * active.
817 * Third trick is to take advantage of the must-be-zero (MBZ) area in
818 * the very beginning of the URB.
819 *
820 * One masks data to be written only for the first channel and uses
821 * offset zero explicitly to land data to the MBZ area avoiding trashing
822 * any other part of the URB.
823 *
824 * Since the WA says that the write needs to be 4 phases long one uses
825 * 4 slots data. All are explicitly zeros in order to to keep the MBZ
826 * area written as zeros.
827 */
828 bld.exec_all().MOV(offset(payload, bld, 1), brw_imm_ud(0x10000u));
829 bld.exec_all().MOV(offset(payload, bld, 2), brw_imm_ud(0u));
830 bld.exec_all().MOV(offset(payload, bld, 3), brw_imm_ud(0u));
831 bld.exec_all().MOV(offset(payload, bld, 4), brw_imm_ud(0u));
832 bld.exec_all().MOV(offset(payload, bld, 5), brw_imm_ud(0u));
833
834 fs_inst *inst = bld.exec_all().emit(SHADER_OPCODE_URB_WRITE_SIMD8_MASKED,
835 reg_undef, payload);
836 inst->eot = true;
837 inst->mlen = 6;
838 inst->offset = 0;
839 }
840 }
841
842 void
emit_cs_terminate()843 fs_visitor::emit_cs_terminate()
844 {
845 assert(devinfo->gen >= 7);
846
847 /* We are getting the thread ID from the compute shader header */
848 assert(stage == MESA_SHADER_COMPUTE || stage == MESA_SHADER_KERNEL);
849
850 /* We can't directly send from g0, since sends with EOT have to use
851 * g112-127. So, copy it to a virtual register, The register allocator will
852 * make sure it uses the appropriate register range.
853 */
854 struct brw_reg g0 = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD);
855 fs_reg payload = fs_reg(VGRF, alloc.allocate(1), BRW_REGISTER_TYPE_UD);
856 bld.group(8, 0).exec_all().MOV(payload, g0);
857
858 /* Send a message to the thread spawner to terminate the thread. */
859 fs_inst *inst = bld.exec_all()
860 .emit(CS_OPCODE_CS_TERMINATE, reg_undef, payload);
861 inst->eot = true;
862 }
863
864 void
emit_barrier()865 fs_visitor::emit_barrier()
866 {
867 uint32_t barrier_id_mask;
868 switch (devinfo->gen) {
869 case 7:
870 case 8:
871 barrier_id_mask = 0x0f000000u; break;
872 case 9:
873 barrier_id_mask = 0x8f000000u; break;
874 case 11:
875 case 12:
876 barrier_id_mask = 0x7f000000u; break;
877 default:
878 unreachable("barrier is only available on gen >= 7");
879 }
880
881 /* We are getting the barrier ID from the compute shader header */
882 assert(stage == MESA_SHADER_COMPUTE || stage == MESA_SHADER_KERNEL);
883
884 fs_reg payload = fs_reg(VGRF, alloc.allocate(1), BRW_REGISTER_TYPE_UD);
885
886 /* Clear the message payload */
887 bld.exec_all().group(8, 0).MOV(payload, brw_imm_ud(0u));
888
889 /* Copy the barrier id from r0.2 to the message payload reg.2 */
890 fs_reg r0_2 = fs_reg(retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD));
891 bld.exec_all().group(1, 0).AND(component(payload, 2), r0_2,
892 brw_imm_ud(barrier_id_mask));
893
894 /* Emit a gateway "barrier" message using the payload we set up, followed
895 * by a wait instruction.
896 */
897 bld.exec_all().emit(SHADER_OPCODE_BARRIER, reg_undef, payload);
898 }
899
fs_visitor(const struct brw_compiler * compiler,void * log_data,void * mem_ctx,const brw_base_prog_key * key,struct brw_stage_prog_data * prog_data,const nir_shader * shader,unsigned dispatch_width,int shader_time_index,const struct brw_vue_map * input_vue_map)900 fs_visitor::fs_visitor(const struct brw_compiler *compiler, void *log_data,
901 void *mem_ctx,
902 const brw_base_prog_key *key,
903 struct brw_stage_prog_data *prog_data,
904 const nir_shader *shader,
905 unsigned dispatch_width,
906 int shader_time_index,
907 const struct brw_vue_map *input_vue_map)
908 : backend_shader(compiler, log_data, mem_ctx, shader, prog_data),
909 key(key), gs_compile(NULL), prog_data(prog_data),
910 input_vue_map(input_vue_map),
911 live_analysis(this), regpressure_analysis(this),
912 performance_analysis(this),
913 dispatch_width(dispatch_width),
914 shader_time_index(shader_time_index),
915 bld(fs_builder(this, dispatch_width).at_end())
916 {
917 init();
918 }
919
fs_visitor(const struct brw_compiler * compiler,void * log_data,void * mem_ctx,struct brw_gs_compile * c,struct brw_gs_prog_data * prog_data,const nir_shader * shader,int shader_time_index)920 fs_visitor::fs_visitor(const struct brw_compiler *compiler, void *log_data,
921 void *mem_ctx,
922 struct brw_gs_compile *c,
923 struct brw_gs_prog_data *prog_data,
924 const nir_shader *shader,
925 int shader_time_index)
926 : backend_shader(compiler, log_data, mem_ctx, shader,
927 &prog_data->base.base),
928 key(&c->key.base), gs_compile(c),
929 prog_data(&prog_data->base.base),
930 live_analysis(this), regpressure_analysis(this),
931 performance_analysis(this),
932 dispatch_width(8),
933 shader_time_index(shader_time_index),
934 bld(fs_builder(this, dispatch_width).at_end())
935 {
936 init();
937 }
938
939
940 void
init()941 fs_visitor::init()
942 {
943 if (key)
944 this->key_tex = &key->tex;
945 else
946 this->key_tex = NULL;
947
948 this->max_dispatch_width = 32;
949 this->prog_data = this->stage_prog_data;
950
951 this->failed = false;
952 this->fail_msg = NULL;
953
954 this->nir_locals = NULL;
955 this->nir_ssa_values = NULL;
956 this->nir_system_values = NULL;
957
958 memset(&this->payload, 0, sizeof(this->payload));
959 this->source_depth_to_render_target = false;
960 this->runtime_check_aads_emit = false;
961 this->first_non_payload_grf = 0;
962 this->max_grf = devinfo->gen >= 7 ? GEN7_MRF_HACK_START : BRW_MAX_GRF;
963
964 this->uniforms = 0;
965 this->last_scratch = 0;
966 this->pull_constant_loc = NULL;
967 this->push_constant_loc = NULL;
968
969 this->shader_stats.scheduler_mode = NULL;
970 this->shader_stats.promoted_constants = 0,
971
972 this->grf_used = 0;
973 this->spilled_any_registers = false;
974 }
975
~fs_visitor()976 fs_visitor::~fs_visitor()
977 {
978 }
979