1 /*
2 * Copyright (C) 2016 Rob Clark <robclark@freedesktop.org>
3 * Copyright © 2018 Google, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 * Authors:
25 * Rob Clark <robclark@freedesktop.org>
26 */
27
28 #include "pipe/p_state.h"
29 #include "util/u_string.h"
30 #include "util/u_memory.h"
31 #include "util/u_prim.h"
32
33 #include "freedreno_state.h"
34 #include "freedreno_resource.h"
35
36 #include "fd6_draw.h"
37 #include "fd6_context.h"
38 #include "fd6_emit.h"
39 #include "fd6_program.h"
40 #include "fd6_format.h"
41 #include "fd6_vsc.h"
42 #include "fd6_zsa.h"
43
44 #include "fd6_pack.h"
45
46 static void
draw_emit_indirect(struct fd_ringbuffer * ring,struct CP_DRAW_INDX_OFFSET_0 * draw0,const struct pipe_draw_info * info,unsigned index_offset)47 draw_emit_indirect(struct fd_ringbuffer *ring,
48 struct CP_DRAW_INDX_OFFSET_0 *draw0,
49 const struct pipe_draw_info *info,
50 unsigned index_offset)
51 {
52 struct fd_resource *ind = fd_resource(info->indirect->buffer);
53
54 if (info->index_size) {
55 struct pipe_resource *idx = info->index.resource;
56 unsigned max_indices = (idx->width0 - index_offset) / info->index_size;
57
58 OUT_PKT(ring, CP_DRAW_INDX_INDIRECT,
59 pack_CP_DRAW_INDX_OFFSET_0(*draw0),
60 A5XX_CP_DRAW_INDX_INDIRECT_INDX_BASE(
61 fd_resource(idx)->bo, index_offset),
62 A5XX_CP_DRAW_INDX_INDIRECT_3(.max_indices = max_indices),
63 A5XX_CP_DRAW_INDX_INDIRECT_INDIRECT(
64 ind->bo, info->indirect->offset)
65 );
66 } else {
67 OUT_PKT(ring, CP_DRAW_INDIRECT,
68 pack_CP_DRAW_INDX_OFFSET_0(*draw0),
69 A5XX_CP_DRAW_INDIRECT_INDIRECT(
70 ind->bo, info->indirect->offset)
71 );
72 }
73 }
74
75 static void
draw_emit(struct fd_ringbuffer * ring,struct CP_DRAW_INDX_OFFSET_0 * draw0,const struct pipe_draw_info * info,unsigned index_offset)76 draw_emit(struct fd_ringbuffer *ring,
77 struct CP_DRAW_INDX_OFFSET_0 *draw0,
78 const struct pipe_draw_info *info,
79 unsigned index_offset)
80 {
81 if (info->index_size) {
82 assert(!info->has_user_indices);
83
84 struct pipe_resource *idx_buffer = info->index.resource;
85 unsigned max_indices = (idx_buffer->width0 - index_offset) / info->index_size;
86
87 OUT_PKT(ring, CP_DRAW_INDX_OFFSET,
88 pack_CP_DRAW_INDX_OFFSET_0(*draw0),
89 CP_DRAW_INDX_OFFSET_1(.num_instances = info->instance_count),
90 CP_DRAW_INDX_OFFSET_2(.num_indices = info->count),
91 CP_DRAW_INDX_OFFSET_3(.first_indx = info->start),
92 A5XX_CP_DRAW_INDX_OFFSET_INDX_BASE(
93 fd_resource(idx_buffer)->bo, index_offset),
94 A5XX_CP_DRAW_INDX_OFFSET_6(.max_indices = max_indices)
95 );
96 } else {
97 OUT_PKT(ring, CP_DRAW_INDX_OFFSET,
98 pack_CP_DRAW_INDX_OFFSET_0(*draw0),
99 CP_DRAW_INDX_OFFSET_1(.num_instances = info->instance_count),
100 CP_DRAW_INDX_OFFSET_2(.num_indices = info->count)
101 );
102 }
103 }
104
105 /* fixup dirty shader state in case some "unrelated" (from the state-
106 * tracker's perspective) state change causes us to switch to a
107 * different variant.
108 */
109 static void
fixup_shader_state(struct fd_context * ctx,struct ir3_shader_key * key)110 fixup_shader_state(struct fd_context *ctx, struct ir3_shader_key *key)
111 {
112 struct fd6_context *fd6_ctx = fd6_context(ctx);
113 struct ir3_shader_key *last_key = &fd6_ctx->last_key;
114
115 if (!ir3_shader_key_equal(last_key, key)) {
116 if (ir3_shader_key_changes_fs(last_key, key)) {
117 ctx->dirty_shader[PIPE_SHADER_FRAGMENT] |= FD_DIRTY_SHADER_PROG;
118 ctx->dirty |= FD_DIRTY_PROG;
119 }
120
121 if (ir3_shader_key_changes_vs(last_key, key)) {
122 ctx->dirty_shader[PIPE_SHADER_VERTEX] |= FD_DIRTY_SHADER_PROG;
123 ctx->dirty |= FD_DIRTY_PROG;
124 }
125
126 fd6_ctx->last_key = *key;
127 }
128 }
129
130 static void
fixup_draw_state(struct fd_context * ctx,struct fd6_emit * emit)131 fixup_draw_state(struct fd_context *ctx, struct fd6_emit *emit)
132 {
133 if (ctx->last.dirty ||
134 (ctx->last.primitive_restart != emit->primitive_restart)) {
135 /* rasterizer state is effected by primitive-restart: */
136 ctx->dirty |= FD_DIRTY_RASTERIZER;
137 ctx->last.primitive_restart = emit->primitive_restart;
138 }
139 }
140
141 static bool
fd6_draw_vbo(struct fd_context * ctx,const struct pipe_draw_info * info,unsigned index_offset)142 fd6_draw_vbo(struct fd_context *ctx, const struct pipe_draw_info *info,
143 unsigned index_offset)
144 {
145 struct fd6_context *fd6_ctx = fd6_context(ctx);
146 struct ir3_shader *gs = ctx->prog.gs;
147 struct fd6_emit emit = {
148 .ctx = ctx,
149 .vtx = &ctx->vtx,
150 .info = info,
151 .key = {
152 .vs = ctx->prog.vs,
153 .gs = ctx->prog.gs,
154 .fs = ctx->prog.fs,
155 .key = {
156 .color_two_side = ctx->rasterizer->light_twoside,
157 .vclamp_color = ctx->rasterizer->clamp_vertex_color,
158 .fclamp_color = ctx->rasterizer->clamp_fragment_color,
159 .rasterflat = ctx->rasterizer->flatshade,
160 .ucp_enables = ctx->rasterizer->clip_plane_enable,
161 .has_per_samp = (fd6_ctx->fsaturate || fd6_ctx->vsaturate),
162 .vsaturate_s = fd6_ctx->vsaturate_s,
163 .vsaturate_t = fd6_ctx->vsaturate_t,
164 .vsaturate_r = fd6_ctx->vsaturate_r,
165 .fsaturate_s = fd6_ctx->fsaturate_s,
166 .fsaturate_t = fd6_ctx->fsaturate_t,
167 .fsaturate_r = fd6_ctx->fsaturate_r,
168 .layer_zero = !gs || !(gs->nir->info.outputs_written & VARYING_BIT_LAYER),
169 .vsamples = ctx->tex[PIPE_SHADER_VERTEX].samples,
170 .fsamples = ctx->tex[PIPE_SHADER_FRAGMENT].samples,
171 .sample_shading = (ctx->min_samples > 1),
172 .msaa = (ctx->framebuffer.samples > 1),
173 },
174 },
175 .rasterflat = ctx->rasterizer->flatshade,
176 .sprite_coord_enable = ctx->rasterizer->sprite_coord_enable,
177 .sprite_coord_mode = ctx->rasterizer->sprite_coord_mode,
178 .primitive_restart = info->primitive_restart && info->index_size,
179 };
180
181 if (!(ctx->prog.vs && ctx->prog.fs))
182 return false;
183
184 if (info->mode == PIPE_PRIM_PATCHES) {
185 emit.key.hs = ctx->prog.hs;
186 emit.key.ds = ctx->prog.ds;
187
188 if (!(ctx->prog.hs && ctx->prog.ds))
189 return false;
190
191 shader_info *ds_info = &emit.key.ds->nir->info;
192 emit.key.key.tessellation = ir3_tess_mode(ds_info->tess.primitive_mode);
193 }
194
195 if (emit.key.gs)
196 emit.key.key.has_gs = true;
197
198 if (!(emit.key.hs || emit.key.ds || emit.key.gs || info->indirect))
199 fd6_vsc_update_sizes(ctx->batch, info);
200
201 fixup_shader_state(ctx, &emit.key.key);
202
203 if (!(ctx->dirty & FD_DIRTY_PROG)) {
204 emit.prog = fd6_ctx->prog;
205 } else {
206 fd6_ctx->prog = fd6_emit_get_prog(&emit);
207 }
208
209 /* bail if compile failed: */
210 if (!fd6_ctx->prog)
211 return false;
212
213 fixup_draw_state(ctx, &emit);
214
215 emit.dirty = ctx->dirty; /* *after* fixup_shader_state() */
216 emit.bs = fd6_emit_get_prog(&emit)->bs;
217 emit.vs = fd6_emit_get_prog(&emit)->vs;
218 emit.hs = fd6_emit_get_prog(&emit)->hs;
219 emit.ds = fd6_emit_get_prog(&emit)->ds;
220 emit.gs = fd6_emit_get_prog(&emit)->gs;
221 emit.fs = fd6_emit_get_prog(&emit)->fs;
222
223 ctx->stats.vs_regs += ir3_shader_halfregs(emit.vs);
224 ctx->stats.hs_regs += COND(emit.hs, ir3_shader_halfregs(emit.hs));
225 ctx->stats.ds_regs += COND(emit.ds, ir3_shader_halfregs(emit.ds));
226 ctx->stats.gs_regs += COND(emit.gs, ir3_shader_halfregs(emit.gs));
227 ctx->stats.fs_regs += ir3_shader_halfregs(emit.fs);
228
229 struct fd_ringbuffer *ring = ctx->batch->draw;
230
231 struct CP_DRAW_INDX_OFFSET_0 draw0 = {
232 .prim_type = ctx->primtypes[info->mode],
233 .vis_cull = USE_VISIBILITY,
234 .gs_enable = !!emit.key.gs,
235 };
236
237 if (info->index_size) {
238 draw0.source_select = DI_SRC_SEL_DMA;
239 draw0.index_size = fd4_size2indextype(info->index_size);
240 } else {
241 draw0.source_select = DI_SRC_SEL_AUTO_INDEX;
242 }
243
244 if (info->mode == PIPE_PRIM_PATCHES) {
245 shader_info *ds_info = &emit.ds->shader->nir->info;
246 uint32_t factor_stride;
247
248 switch (ds_info->tess.primitive_mode) {
249 case GL_ISOLINES:
250 draw0.patch_type = TESS_ISOLINES;
251 factor_stride = 12;
252 break;
253 case GL_TRIANGLES:
254 draw0.patch_type = TESS_TRIANGLES;
255 factor_stride = 20;
256 break;
257 case GL_QUADS:
258 draw0.patch_type = TESS_QUADS;
259 factor_stride = 28;
260 break;
261 default:
262 unreachable("bad tessmode");
263 }
264
265 draw0.prim_type = DI_PT_PATCHES0 + info->vertices_per_patch;
266 draw0.tess_enable = true;
267
268 ctx->batch->tessellation = true;
269 ctx->batch->tessparam_size = MAX2(ctx->batch->tessparam_size,
270 emit.hs->output_size * 4 * info->count);
271 ctx->batch->tessfactor_size = MAX2(ctx->batch->tessfactor_size,
272 factor_stride * info->count);
273
274 if (!ctx->batch->tess_addrs_constobj) {
275 /* Reserve space for the bo address - we'll write them later in
276 * setup_tess_buffers(). We need 2 bo address, but indirect
277 * constant upload needs at least 4 vec4s.
278 */
279 unsigned size = 4 * 16;
280
281 ctx->batch->tess_addrs_constobj = fd_submit_new_ringbuffer(
282 ctx->batch->submit, size, FD_RINGBUFFER_STREAMING);
283
284 ctx->batch->tess_addrs_constobj->cur += size;
285 }
286 }
287
288 uint32_t index_start = info->index_size ? info->index_bias : info->start;
289 if (ctx->last.dirty || (ctx->last.index_start != index_start)) {
290 OUT_PKT4(ring, REG_A6XX_VFD_INDEX_OFFSET, 1);
291 OUT_RING(ring, index_start); /* VFD_INDEX_OFFSET */
292 ctx->last.index_start = index_start;
293 }
294
295 if (ctx->last.dirty || (ctx->last.instance_start != info->start_instance)) {
296 OUT_PKT4(ring, REG_A6XX_VFD_INSTANCE_START_OFFSET, 1);
297 OUT_RING(ring, info->start_instance); /* VFD_INSTANCE_START_OFFSET */
298 ctx->last.instance_start = info->start_instance;
299 }
300
301 uint32_t restart_index = info->primitive_restart ? info->restart_index : 0xffffffff;
302 if (ctx->last.dirty || (ctx->last.restart_index != restart_index)) {
303 OUT_PKT4(ring, REG_A6XX_PC_RESTART_INDEX, 1);
304 OUT_RING(ring, restart_index); /* PC_RESTART_INDEX */
305 ctx->last.restart_index = restart_index;
306 }
307
308 fd6_emit_state(ring, &emit);
309
310 /* for debug after a lock up, write a unique counter value
311 * to scratch7 for each draw, to make it easier to match up
312 * register dumps to cmdstream. The combination of IB
313 * (scratch6) and DRAW is enough to "triangulate" the
314 * particular draw that caused lockup.
315 */
316 emit_marker6(ring, 7);
317
318 if (info->indirect) {
319 draw_emit_indirect(ring, &draw0, info, index_offset);
320 } else {
321 draw_emit(ring, &draw0, info, index_offset);
322 }
323
324 emit_marker6(ring, 7);
325 fd_reset_wfi(ctx->batch);
326
327 if (emit.streamout_mask) {
328 struct fd_ringbuffer *ring = ctx->batch->draw;
329
330 for (unsigned i = 0; i < PIPE_MAX_SO_BUFFERS; i++) {
331 if (emit.streamout_mask & (1 << i)) {
332 fd6_event_write(ctx->batch, ring, FLUSH_SO_0 + i, false);
333 }
334 }
335 }
336
337 fd_context_all_clean(ctx);
338
339 return true;
340 }
341
342 static void
fd6_clear_lrz(struct fd_batch * batch,struct fd_resource * zsbuf,double depth)343 fd6_clear_lrz(struct fd_batch *batch, struct fd_resource *zsbuf, double depth)
344 {
345 struct fd_ringbuffer *ring;
346 struct fd_screen *screen = batch->ctx->screen;
347
348 ring = fd_batch_get_prologue(batch);
349
350 emit_marker6(ring, 7);
351 OUT_PKT7(ring, CP_SET_MARKER, 1);
352 OUT_RING(ring, A6XX_CP_SET_MARKER_0_MODE(RM6_BYPASS));
353 emit_marker6(ring, 7);
354
355 OUT_WFI5(ring);
356
357 OUT_REG(ring, A6XX_RB_CCU_CNTL(.offset = screen->info.a6xx.ccu_offset_bypass));
358
359 OUT_REG(ring, A6XX_HLSQ_INVALIDATE_CMD(
360 .vs_state = true,
361 .hs_state = true,
362 .ds_state = true,
363 .gs_state = true,
364 .fs_state = true,
365 .cs_state = true,
366 .gfx_ibo = true,
367 .cs_ibo = true,
368 .gfx_shared_const = true,
369 .gfx_bindless = 0x1f,
370 .cs_bindless = 0x1f
371 ));
372
373 emit_marker6(ring, 7);
374 OUT_PKT7(ring, CP_SET_MARKER, 1);
375 OUT_RING(ring, A6XX_CP_SET_MARKER_0_MODE(RM6_BLIT2DSCALE));
376 emit_marker6(ring, 7);
377
378 OUT_PKT4(ring, REG_A6XX_RB_2D_UNKNOWN_8C01, 1);
379 OUT_RING(ring, 0x0);
380
381 OUT_PKT4(ring, REG_A6XX_SP_PS_2D_SRC_INFO, 13);
382 OUT_RING(ring, 0x00000000);
383 OUT_RING(ring, 0x00000000);
384 OUT_RING(ring, 0x00000000);
385 OUT_RING(ring, 0x00000000);
386 OUT_RING(ring, 0x00000000);
387 OUT_RING(ring, 0x00000000);
388 OUT_RING(ring, 0x00000000);
389 OUT_RING(ring, 0x00000000);
390 OUT_RING(ring, 0x00000000);
391 OUT_RING(ring, 0x00000000);
392 OUT_RING(ring, 0x00000000);
393 OUT_RING(ring, 0x00000000);
394 OUT_RING(ring, 0x00000000);
395
396 OUT_PKT4(ring, REG_A6XX_SP_2D_DST_FORMAT, 1);
397 OUT_RING(ring, 0x0000f410);
398
399 OUT_PKT4(ring, REG_A6XX_GRAS_2D_BLIT_CNTL, 1);
400 OUT_RING(ring, A6XX_GRAS_2D_BLIT_CNTL_COLOR_FORMAT(FMT6_16_UNORM) |
401 0x4f00080);
402
403 OUT_PKT4(ring, REG_A6XX_RB_2D_BLIT_CNTL, 1);
404 OUT_RING(ring, A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT(FMT6_16_UNORM) |
405 0x4f00080);
406
407 fd6_event_write(batch, ring, PC_CCU_FLUSH_COLOR_TS, true);
408 fd6_event_write(batch, ring, PC_CCU_INVALIDATE_COLOR, false);
409
410 OUT_PKT4(ring, REG_A6XX_RB_2D_SRC_SOLID_C0, 4);
411 OUT_RING(ring, fui(depth));
412 OUT_RING(ring, 0x00000000);
413 OUT_RING(ring, 0x00000000);
414 OUT_RING(ring, 0x00000000);
415
416 OUT_PKT4(ring, REG_A6XX_RB_2D_DST_INFO, 9);
417 OUT_RING(ring, A6XX_RB_2D_DST_INFO_COLOR_FORMAT(FMT6_16_UNORM) |
418 A6XX_RB_2D_DST_INFO_TILE_MODE(TILE6_LINEAR) |
419 A6XX_RB_2D_DST_INFO_COLOR_SWAP(WZYX));
420 OUT_RELOC(ring, zsbuf->lrz, 0, 0, 0);
421 OUT_RING(ring, A6XX_RB_2D_DST_PITCH(zsbuf->lrz_pitch * 2).value);
422 OUT_RING(ring, 0x00000000);
423 OUT_RING(ring, 0x00000000);
424 OUT_RING(ring, 0x00000000);
425 OUT_RING(ring, 0x00000000);
426 OUT_RING(ring, 0x00000000);
427
428 OUT_REG(ring,
429 A6XX_GRAS_2D_SRC_TL_X(0),
430 A6XX_GRAS_2D_SRC_BR_X(0),
431 A6XX_GRAS_2D_SRC_TL_Y(0),
432 A6XX_GRAS_2D_SRC_BR_Y(0));
433
434 OUT_PKT4(ring, REG_A6XX_GRAS_2D_DST_TL, 2);
435 OUT_RING(ring, A6XX_GRAS_2D_DST_TL_X(0) |
436 A6XX_GRAS_2D_DST_TL_Y(0));
437 OUT_RING(ring, A6XX_GRAS_2D_DST_BR_X(zsbuf->lrz_width - 1) |
438 A6XX_GRAS_2D_DST_BR_Y(zsbuf->lrz_height - 1));
439
440 fd6_event_write(batch, ring, 0x3f, false);
441
442 OUT_WFI5(ring);
443
444 OUT_PKT4(ring, REG_A6XX_RB_UNKNOWN_8E04, 1);
445 OUT_RING(ring, screen->info.a6xx.magic.RB_UNKNOWN_8E04_blit);
446
447 OUT_PKT7(ring, CP_BLIT, 1);
448 OUT_RING(ring, CP_BLIT_0_OP(BLIT_OP_SCALE));
449
450 OUT_WFI5(ring);
451
452 OUT_PKT4(ring, REG_A6XX_RB_UNKNOWN_8E04, 1);
453 OUT_RING(ring, 0x0); /* RB_UNKNOWN_8E04 */
454
455 fd6_event_write(batch, ring, PC_CCU_FLUSH_COLOR_TS, true);
456 fd6_event_write(batch, ring, PC_CCU_FLUSH_DEPTH_TS, true);
457 fd6_event_write(batch, ring, CACHE_FLUSH_TS, true);
458
459 fd6_cache_inv(batch, ring);
460 }
461
is_z32(enum pipe_format format)462 static bool is_z32(enum pipe_format format)
463 {
464 switch (format) {
465 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
466 case PIPE_FORMAT_Z32_UNORM:
467 case PIPE_FORMAT_Z32_FLOAT:
468 return true;
469 default:
470 return false;
471 }
472 }
473
474 static bool
fd6_clear(struct fd_context * ctx,unsigned buffers,const union pipe_color_union * color,double depth,unsigned stencil)475 fd6_clear(struct fd_context *ctx, unsigned buffers,
476 const union pipe_color_union *color, double depth, unsigned stencil)
477 {
478 struct pipe_framebuffer_state *pfb = &ctx->batch->framebuffer;
479 const bool has_depth = pfb->zsbuf;
480 unsigned color_buffers = buffers >> 2;
481
482 /* we need to do multisample clear on 3d pipe, so fallback to u_blitter: */
483 if (pfb->samples > 1)
484 return false;
485
486 /* If we're clearing after draws, fallback to 3D pipe clears. We could
487 * use blitter clears in the draw batch but then we'd have to patch up the
488 * gmem offsets. This doesn't seem like a useful thing to optimize for
489 * however.*/
490 if (ctx->batch->num_draws > 0)
491 return false;
492
493 foreach_bit(i, color_buffers)
494 ctx->batch->clear_color[i] = *color;
495 if (buffers & PIPE_CLEAR_DEPTH)
496 ctx->batch->clear_depth = depth;
497 if (buffers & PIPE_CLEAR_STENCIL)
498 ctx->batch->clear_stencil = stencil;
499
500 ctx->batch->fast_cleared |= buffers;
501
502 if (has_depth && (buffers & PIPE_CLEAR_DEPTH)) {
503 struct fd_resource *zsbuf = fd_resource(pfb->zsbuf->texture);
504 if (zsbuf->lrz && !is_z32(pfb->zsbuf->format)) {
505 zsbuf->lrz_valid = true;
506 zsbuf->lrz_direction = FD_LRZ_UNKNOWN;
507 fd6_clear_lrz(ctx->batch, zsbuf, depth);
508 }
509 }
510
511 return true;
512 }
513
514 void
fd6_draw_init(struct pipe_context * pctx)515 fd6_draw_init(struct pipe_context *pctx)
516 {
517 struct fd_context *ctx = fd_context(pctx);
518 ctx->draw_vbo = fd6_draw_vbo;
519 ctx->clear = fd6_clear;
520 }
521