1 /*
2 * Copyright (C) 2019-2020 Collabora, Ltd.
3 * © Copyright 2018 Alyssa Rosenzweig
4 * Copyright © 2014-2017 Broadcom
5 * Copyright (C) 2017 Intel Corporation
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 * SOFTWARE.
25 *
26 */
27
28 #include <sys/poll.h>
29 #include <errno.h>
30
31 #include "pan_bo.h"
32 #include "pan_context.h"
33 #include "pan_minmax_cache.h"
34 #include "panfrost-quirks.h"
35
36 #include "util/macros.h"
37 #include "util/format/u_format.h"
38 #include "util/u_inlines.h"
39 #include "util/u_upload_mgr.h"
40 #include "util/u_memory.h"
41 #include "util/u_vbuf.h"
42 #include "util/half_float.h"
43 #include "util/u_helpers.h"
44 #include "util/format/u_format.h"
45 #include "util/u_prim.h"
46 #include "util/u_prim_restart.h"
47 #include "indices/u_primconvert.h"
48 #include "tgsi/tgsi_parse.h"
49 #include "tgsi/tgsi_from_mesa.h"
50 #include "util/u_math.h"
51
52 #include "midgard_pack.h"
53 #include "pan_screen.h"
54 #include "pan_blending.h"
55 #include "pan_blend_shaders.h"
56 #include "pan_cmdstream.h"
57 #include "pan_util.h"
58 #include "decode.h"
59 #include "util/pan_lower_framebuffer.h"
60
61 void
panfrost_emit_midg_tiler(struct panfrost_batch * batch,struct mali_midgard_tiler_packed * tp,unsigned vertex_count)62 panfrost_emit_midg_tiler(struct panfrost_batch *batch,
63 struct mali_midgard_tiler_packed *tp,
64 unsigned vertex_count)
65 {
66 struct panfrost_device *device = pan_device(batch->ctx->base.screen);
67 bool hierarchy = !(device->quirks & MIDGARD_NO_HIER_TILING);
68 unsigned height = batch->key.height;
69 unsigned width = batch->key.width;
70
71 pan_pack(tp, MIDGARD_TILER, t) {
72 t.hierarchy_mask =
73 panfrost_choose_hierarchy_mask(width, height,
74 vertex_count, hierarchy);
75
76 /* Compute the polygon header size and use that to offset the body */
77
78 unsigned header_size =
79 panfrost_tiler_header_size(width, height,
80 t.hierarchy_mask, hierarchy);
81
82 t.polygon_list_size =
83 panfrost_tiler_full_size(width, height, t.hierarchy_mask,
84 hierarchy);
85
86 if (vertex_count) {
87 t.polygon_list =
88 panfrost_batch_get_polygon_list(batch,
89 header_size +
90 t.polygon_list_size);
91
92 t.heap_start = device->tiler_heap->ptr.gpu;
93 t.heap_end = device->tiler_heap->ptr.gpu +
94 device->tiler_heap->size;
95 } else {
96 struct panfrost_bo *tiler_dummy;
97
98 tiler_dummy = panfrost_batch_get_tiler_dummy(batch);
99 header_size = MALI_MIDGARD_TILER_MINIMUM_HEADER_SIZE;
100
101 /* The tiler is disabled, so don't allow the tiler heap */
102 t.heap_start = tiler_dummy->ptr.gpu;
103 t.heap_end = t.heap_start;
104
105 /* Use a dummy polygon list */
106 t.polygon_list = tiler_dummy->ptr.gpu;
107
108 /* Disable the tiler */
109 if (hierarchy)
110 t.hierarchy_mask |= MALI_MIDGARD_TILER_DISABLED;
111 else {
112 t.hierarchy_mask = MALI_MIDGARD_TILER_USER;
113 t.polygon_list_size = MALI_MIDGARD_TILER_MINIMUM_HEADER_SIZE + 4;
114
115 /* We don't have a WRITE_VALUE job, so write the polygon list manually */
116 uint32_t *polygon_list_body = (uint32_t *) (tiler_dummy->ptr.cpu + header_size);
117 polygon_list_body[0] = 0xa0000000; /* TODO: Just that? */
118 }
119 }
120 t.polygon_list_body = t.polygon_list + header_size;
121 }
122 }
123
124 static void
panfrost_clear(struct pipe_context * pipe,unsigned buffers,const struct pipe_scissor_state * scissor_state,const union pipe_color_union * color,double depth,unsigned stencil)125 panfrost_clear(
126 struct pipe_context *pipe,
127 unsigned buffers,
128 const struct pipe_scissor_state *scissor_state,
129 const union pipe_color_union *color,
130 double depth, unsigned stencil)
131 {
132 struct panfrost_context *ctx = pan_context(pipe);
133
134 /* TODO: panfrost_get_fresh_batch_for_fbo() instantiates a new batch if
135 * the existing batch targeting this FBO has draws. We could probably
136 * avoid that by replacing plain clears by quad-draws with a specific
137 * color/depth/stencil value, thus avoiding the generation of extra
138 * fragment jobs.
139 */
140 struct panfrost_batch *batch = panfrost_get_fresh_batch_for_fbo(ctx);
141 panfrost_batch_clear(batch, buffers, color, depth, stencil);
142 }
143
144 bool
panfrost_writes_point_size(struct panfrost_context * ctx)145 panfrost_writes_point_size(struct panfrost_context *ctx)
146 {
147 assert(ctx->shader[PIPE_SHADER_VERTEX]);
148 struct panfrost_shader_state *vs = panfrost_get_shader_state(ctx, PIPE_SHADER_VERTEX);
149
150 return vs->writes_point_size && ctx->active_prim == PIPE_PRIM_POINTS;
151 }
152
153 /* The entire frame is in memory -- send it off to the kernel! */
154
155 void
panfrost_flush(struct pipe_context * pipe,struct pipe_fence_handle ** fence,unsigned flags)156 panfrost_flush(
157 struct pipe_context *pipe,
158 struct pipe_fence_handle **fence,
159 unsigned flags)
160 {
161 struct panfrost_context *ctx = pan_context(pipe);
162 struct panfrost_device *dev = pan_device(pipe->screen);
163
164
165 /* Submit all pending jobs */
166 panfrost_flush_all_batches(ctx);
167
168 if (fence) {
169 struct panfrost_fence *f = panfrost_fence_create(ctx);
170 pipe->screen->fence_reference(pipe->screen, fence, NULL);
171 *fence = (struct pipe_fence_handle *)f;
172 }
173
174 if (dev->debug & PAN_DBG_TRACE)
175 pandecode_next_frame();
176 }
177
178 static void
panfrost_texture_barrier(struct pipe_context * pipe,unsigned flags)179 panfrost_texture_barrier(struct pipe_context *pipe, unsigned flags)
180 {
181 struct panfrost_context *ctx = pan_context(pipe);
182 panfrost_flush_all_batches(ctx);
183 }
184
185 #define DEFINE_CASE(c) case PIPE_PRIM_##c: return MALI_DRAW_MODE_##c;
186
187 static int
pan_draw_mode(enum pipe_prim_type mode)188 pan_draw_mode(enum pipe_prim_type mode)
189 {
190 switch (mode) {
191 DEFINE_CASE(POINTS);
192 DEFINE_CASE(LINES);
193 DEFINE_CASE(LINE_LOOP);
194 DEFINE_CASE(LINE_STRIP);
195 DEFINE_CASE(TRIANGLES);
196 DEFINE_CASE(TRIANGLE_STRIP);
197 DEFINE_CASE(TRIANGLE_FAN);
198 DEFINE_CASE(QUADS);
199 DEFINE_CASE(QUAD_STRIP);
200 DEFINE_CASE(POLYGON);
201
202 default:
203 unreachable("Invalid draw mode");
204 }
205 }
206
207 #undef DEFINE_CASE
208
209 static bool
panfrost_scissor_culls_everything(struct panfrost_context * ctx)210 panfrost_scissor_culls_everything(struct panfrost_context *ctx)
211 {
212 const struct pipe_scissor_state *ss = &ctx->scissor;
213
214 /* Check if we're scissoring at all */
215
216 if (!ctx->rasterizer->base.scissor)
217 return false;
218
219 return (ss->minx == ss->maxx) || (ss->miny == ss->maxy);
220 }
221
222 /* Count generated primitives (when there is no geom/tess shaders) for
223 * transform feedback */
224
225 static void
panfrost_statistics_record(struct panfrost_context * ctx,const struct pipe_draw_info * info)226 panfrost_statistics_record(
227 struct panfrost_context *ctx,
228 const struct pipe_draw_info *info)
229 {
230 if (!ctx->active_queries)
231 return;
232
233 uint32_t prims = u_prims_for_vertices(info->mode, info->count);
234 ctx->prims_generated += prims;
235
236 if (!ctx->streamout.num_targets)
237 return;
238
239 ctx->tf_prims_generated += prims;
240 }
241
242 static void
panfrost_update_streamout_offsets(struct panfrost_context * ctx)243 panfrost_update_streamout_offsets(struct panfrost_context *ctx)
244 {
245 for (unsigned i = 0; i < ctx->streamout.num_targets; ++i) {
246 unsigned count;
247
248 count = u_stream_outputs_for_vertices(ctx->active_prim,
249 ctx->vertex_count);
250 pan_so_target(ctx->streamout.targets[i])->offset += count;
251 }
252 }
253
254 static inline void
pan_emit_draw_descs(struct panfrost_batch * batch,struct MALI_DRAW * d,enum pipe_shader_type st)255 pan_emit_draw_descs(struct panfrost_batch *batch,
256 struct MALI_DRAW *d, enum pipe_shader_type st)
257 {
258 d->offset_start = batch->ctx->offset_start;
259 d->instance_size = batch->ctx->instance_count > 1 ?
260 batch->ctx->padded_count : 1;
261
262 d->uniform_buffers = panfrost_emit_const_buf(batch, st, &d->push_uniforms);
263 d->textures = panfrost_emit_texture_descriptors(batch, st);
264 d->samplers = panfrost_emit_sampler_descriptors(batch, st);
265 }
266
267 static enum mali_index_type
panfrost_translate_index_size(unsigned size)268 panfrost_translate_index_size(unsigned size)
269 {
270 switch (size) {
271 case 1: return MALI_INDEX_TYPE_UINT8;
272 case 2: return MALI_INDEX_TYPE_UINT16;
273 case 4: return MALI_INDEX_TYPE_UINT32;
274 default: unreachable("Invalid index size");
275 }
276 }
277
278 static void
panfrost_draw_emit_vertex(struct panfrost_batch * batch,const struct pipe_draw_info * info,void * invocation_template,mali_ptr shared_mem,mali_ptr vs_vary,mali_ptr varyings,void * job)279 panfrost_draw_emit_vertex(struct panfrost_batch *batch,
280 const struct pipe_draw_info *info,
281 void *invocation_template,
282 mali_ptr shared_mem, mali_ptr vs_vary,
283 mali_ptr varyings, void *job)
284 {
285 struct panfrost_context *ctx = batch->ctx;
286 struct panfrost_device *device = pan_device(ctx->base.screen);
287
288 void *section =
289 pan_section_ptr(job, COMPUTE_JOB, INVOCATION);
290 memcpy(section, invocation_template, MALI_INVOCATION_LENGTH);
291
292 pan_section_pack(job, COMPUTE_JOB, PARAMETERS, cfg) {
293 cfg.job_task_split = 5;
294 }
295
296 pan_section_pack(job, COMPUTE_JOB, DRAW, cfg) {
297 cfg.draw_descriptor_is_64b = true;
298 if (!(device->quirks & IS_BIFROST))
299 cfg.texture_descriptor_is_64b = true;
300 cfg.state = panfrost_emit_compute_shader_meta(batch, PIPE_SHADER_VERTEX);
301 cfg.attributes = panfrost_emit_vertex_data(batch, &cfg.attribute_buffers);
302 cfg.varyings = vs_vary;
303 cfg.varying_buffers = varyings;
304 cfg.thread_storage = shared_mem;
305 pan_emit_draw_descs(batch, &cfg, PIPE_SHADER_VERTEX);
306 }
307 }
308
309 static void
panfrost_emit_primitive_size(struct panfrost_context * ctx,bool points,mali_ptr size_array,void * prim_size)310 panfrost_emit_primitive_size(struct panfrost_context *ctx,
311 bool points, mali_ptr size_array,
312 void *prim_size)
313 {
314 struct panfrost_rasterizer *rast = ctx->rasterizer;
315
316 pan_pack(prim_size, PRIMITIVE_SIZE, cfg) {
317 if (panfrost_writes_point_size(ctx)) {
318 cfg.size_array = size_array;
319 } else {
320 cfg.constant = points ?
321 rast->base.point_size :
322 rast->base.line_width;
323 }
324 }
325 }
326
327 static void
panfrost_draw_emit_tiler(struct panfrost_batch * batch,const struct pipe_draw_info * info,void * invocation_template,mali_ptr shared_mem,mali_ptr indices,mali_ptr fs_vary,mali_ptr varyings,mali_ptr pos,mali_ptr psiz,void * job)328 panfrost_draw_emit_tiler(struct panfrost_batch *batch,
329 const struct pipe_draw_info *info,
330 void *invocation_template,
331 mali_ptr shared_mem, mali_ptr indices,
332 mali_ptr fs_vary, mali_ptr varyings,
333 mali_ptr pos, mali_ptr psiz, void *job)
334 {
335 struct panfrost_context *ctx = batch->ctx;
336 struct pipe_rasterizer_state *rast = &ctx->rasterizer->base;
337 struct panfrost_device *device = pan_device(ctx->base.screen);
338 bool is_bifrost = device->quirks & IS_BIFROST;
339
340 void *section = is_bifrost ?
341 pan_section_ptr(job, BIFROST_TILER_JOB, INVOCATION) :
342 pan_section_ptr(job, MIDGARD_TILER_JOB, INVOCATION);
343 memcpy(section, invocation_template, MALI_INVOCATION_LENGTH);
344
345 section = is_bifrost ?
346 pan_section_ptr(job, BIFROST_TILER_JOB, PRIMITIVE) :
347 pan_section_ptr(job, MIDGARD_TILER_JOB, PRIMITIVE);
348 pan_pack(section, PRIMITIVE, cfg) {
349 cfg.draw_mode = pan_draw_mode(info->mode);
350 if (panfrost_writes_point_size(ctx))
351 cfg.point_size_array_format = MALI_POINT_SIZE_ARRAY_FORMAT_FP16;
352 cfg.first_provoking_vertex = rast->flatshade_first;
353 if (info->primitive_restart)
354 cfg.primitive_restart = MALI_PRIMITIVE_RESTART_IMPLICIT;
355 cfg.job_task_split = 6;
356
357 if (info->index_size) {
358 cfg.index_type = panfrost_translate_index_size(info->index_size);
359 cfg.indices = indices;
360 cfg.base_vertex_offset = info->index_bias - ctx->offset_start;
361 cfg.index_count = info->count;
362 } else {
363 cfg.index_count = info->count_from_stream_output ?
364 pan_so_target(info->count_from_stream_output)->offset :
365 ctx->vertex_count;
366 }
367 }
368
369 bool points = info->mode == PIPE_PRIM_POINTS;
370 void *prim_size = is_bifrost ?
371 pan_section_ptr(job, BIFROST_TILER_JOB, PRIMITIVE_SIZE) :
372 pan_section_ptr(job, MIDGARD_TILER_JOB, PRIMITIVE_SIZE);
373
374 if (is_bifrost) {
375 panfrost_emit_primitive_size(ctx, points, psiz, prim_size);
376 pan_section_pack(job, BIFROST_TILER_JOB, TILER, cfg) {
377 cfg.address = panfrost_batch_get_bifrost_tiler(batch, ~0);
378 }
379 pan_section_pack(job, BIFROST_TILER_JOB, PADDING, padding) {}
380 }
381
382 section = is_bifrost ?
383 pan_section_ptr(job, BIFROST_TILER_JOB, DRAW) :
384 pan_section_ptr(job, MIDGARD_TILER_JOB, DRAW);
385 pan_pack(section, DRAW, cfg) {
386 cfg.four_components_per_vertex = true;
387 cfg.draw_descriptor_is_64b = true;
388 if (!(device->quirks & IS_BIFROST))
389 cfg.texture_descriptor_is_64b = true;
390 cfg.front_face_ccw = rast->front_ccw;
391 cfg.cull_front_face = rast->cull_face & PIPE_FACE_FRONT;
392 cfg.cull_back_face = rast->cull_face & PIPE_FACE_BACK;
393 cfg.position = pos;
394 cfg.state = panfrost_emit_frag_shader_meta(batch);
395 cfg.viewport = panfrost_emit_viewport(batch);
396 cfg.varyings = fs_vary;
397 cfg.varying_buffers = varyings;
398 cfg.thread_storage = shared_mem;
399
400 pan_emit_draw_descs(batch, &cfg, PIPE_SHADER_FRAGMENT);
401
402 if (ctx->occlusion_query && ctx->active_queries) {
403 if (ctx->occlusion_query->type == PIPE_QUERY_OCCLUSION_COUNTER)
404 cfg.occlusion_query = MALI_OCCLUSION_MODE_COUNTER;
405 else
406 cfg.occlusion_query = MALI_OCCLUSION_MODE_PREDICATE;
407 cfg.occlusion = ctx->occlusion_query->bo->ptr.gpu;
408 panfrost_batch_add_bo(ctx->batch, ctx->occlusion_query->bo,
409 PAN_BO_ACCESS_SHARED |
410 PAN_BO_ACCESS_RW |
411 PAN_BO_ACCESS_FRAGMENT);
412 }
413 }
414
415 if (!is_bifrost)
416 panfrost_emit_primitive_size(ctx, points, psiz, prim_size);
417 }
418
419 static void
panfrost_draw_vbo(struct pipe_context * pipe,const struct pipe_draw_info * info)420 panfrost_draw_vbo(
421 struct pipe_context *pipe,
422 const struct pipe_draw_info *info)
423 {
424 struct panfrost_context *ctx = pan_context(pipe);
425 struct panfrost_device *device = pan_device(ctx->base.screen);
426
427 /* First of all, check the scissor to see if anything is drawn at all.
428 * If it's not, we drop the draw (mostly a conformance issue;
429 * well-behaved apps shouldn't hit this) */
430
431 if (panfrost_scissor_culls_everything(ctx))
432 return;
433
434 int mode = info->mode;
435
436 /* Fallback unsupported restart index */
437 unsigned primitive_index = (1 << (info->index_size * 8)) - 1;
438
439 if (info->primitive_restart && info->index_size
440 && info->restart_index != primitive_index) {
441 util_draw_vbo_without_prim_restart(pipe, info);
442 return;
443 }
444
445 /* Fallback for unsupported modes */
446
447 assert(ctx->rasterizer != NULL);
448
449 if (!(ctx->draw_modes & (1 << mode))) {
450 if (info->count < 4) {
451 /* Degenerate case? */
452 return;
453 }
454
455 util_primconvert_save_rasterizer_state(ctx->primconvert, &ctx->rasterizer->base);
456 util_primconvert_draw_vbo(ctx->primconvert, info);
457 return;
458 }
459
460 /* Now that we have a guaranteed terminating path, find the job. */
461
462 struct panfrost_batch *batch = panfrost_get_batch_for_fbo(ctx);
463 panfrost_batch_set_requirements(batch);
464
465 /* Take into account a negative bias */
466 ctx->vertex_count = info->count + abs(info->index_bias);
467 ctx->instance_count = info->instance_count;
468 ctx->active_prim = info->mode;
469
470 bool is_bifrost = device->quirks & IS_BIFROST;
471 struct panfrost_ptr tiler =
472 panfrost_pool_alloc_aligned(&batch->pool,
473 is_bifrost ?
474 MALI_BIFROST_TILER_JOB_LENGTH :
475 MALI_MIDGARD_TILER_JOB_LENGTH,
476 64);
477 struct panfrost_ptr vertex =
478 panfrost_pool_alloc_aligned(&batch->pool,
479 MALI_COMPUTE_JOB_LENGTH,
480 64);
481
482 unsigned vertex_count = ctx->vertex_count;
483
484 mali_ptr shared_mem = panfrost_batch_reserve_framebuffer(batch);
485
486 unsigned min_index = 0, max_index = 0;
487 mali_ptr indices = 0;
488
489 if (info->index_size) {
490 indices = panfrost_get_index_buffer_bounded(ctx, info,
491 &min_index,
492 &max_index);
493
494 /* Use the corresponding values */
495 vertex_count = max_index - min_index + 1;
496 ctx->offset_start = min_index + info->index_bias;
497 } else {
498 ctx->offset_start = info->start;
499 }
500
501 /* Encode the padded vertex count */
502
503 if (info->instance_count > 1)
504 ctx->padded_count = panfrost_padded_vertex_count(vertex_count);
505 else
506 ctx->padded_count = vertex_count;
507
508 panfrost_statistics_record(ctx, info);
509
510 struct mali_invocation_packed invocation;
511 panfrost_pack_work_groups_compute(&invocation,
512 1, vertex_count, info->instance_count,
513 1, 1, 1, true);
514
515 /* Emit all sort of descriptors. */
516 mali_ptr varyings = 0, vs_vary = 0, fs_vary = 0, pos = 0, psiz = 0;
517
518 panfrost_emit_varying_descriptor(batch,
519 ctx->padded_count *
520 ctx->instance_count,
521 &vs_vary, &fs_vary, &varyings,
522 &pos, &psiz);
523
524 /* Fire off the draw itself */
525 panfrost_draw_emit_vertex(batch, info, &invocation, shared_mem,
526 vs_vary, varyings, vertex.cpu);
527 panfrost_draw_emit_tiler(batch, info, &invocation, shared_mem, indices,
528 fs_vary, varyings, pos, psiz, tiler.cpu);
529 panfrost_emit_vertex_tiler_jobs(batch, &vertex, &tiler);
530
531 /* Adjust the batch stack size based on the new shader stack sizes. */
532 panfrost_batch_adjust_stack_size(batch);
533
534 /* Increment transform feedback offsets */
535 panfrost_update_streamout_offsets(ctx);
536 }
537
538 /* CSO state */
539
540 static void
panfrost_generic_cso_delete(struct pipe_context * pctx,void * hwcso)541 panfrost_generic_cso_delete(struct pipe_context *pctx, void *hwcso)
542 {
543 free(hwcso);
544 }
545
546 static void *
panfrost_create_rasterizer_state(struct pipe_context * pctx,const struct pipe_rasterizer_state * cso)547 panfrost_create_rasterizer_state(
548 struct pipe_context *pctx,
549 const struct pipe_rasterizer_state *cso)
550 {
551 struct panfrost_rasterizer *so = CALLOC_STRUCT(panfrost_rasterizer);
552
553 so->base = *cso;
554
555 /* Gauranteed with the core GL call, so don't expose ARB_polygon_offset */
556 assert(cso->offset_clamp == 0.0);
557
558 return so;
559 }
560
561 static void
panfrost_bind_rasterizer_state(struct pipe_context * pctx,void * hwcso)562 panfrost_bind_rasterizer_state(
563 struct pipe_context *pctx,
564 void *hwcso)
565 {
566 struct panfrost_context *ctx = pan_context(pctx);
567 ctx->rasterizer = hwcso;
568 }
569
570 static void *
panfrost_create_vertex_elements_state(struct pipe_context * pctx,unsigned num_elements,const struct pipe_vertex_element * elements)571 panfrost_create_vertex_elements_state(
572 struct pipe_context *pctx,
573 unsigned num_elements,
574 const struct pipe_vertex_element *elements)
575 {
576 struct panfrost_vertex_state *so = CALLOC_STRUCT(panfrost_vertex_state);
577 struct panfrost_device *dev = pan_device(pctx->screen);
578
579 so->num_elements = num_elements;
580 memcpy(so->pipe, elements, sizeof(*elements) * num_elements);
581
582 for (int i = 0; i < num_elements; ++i) {
583 enum pipe_format fmt = elements[i].src_format;
584 const struct util_format_description *desc = util_format_description(fmt);
585 so->formats[i] = dev->formats[desc->format].hw;
586 assert(so->formats[i]);
587 }
588
589 /* Let's also prepare vertex builtins */
590 so->formats[PAN_VERTEX_ID] = dev->formats[PIPE_FORMAT_R32_UINT].hw;
591 so->formats[PAN_INSTANCE_ID] = dev->formats[PIPE_FORMAT_R32_UINT].hw;
592
593 return so;
594 }
595
596 static void
panfrost_bind_vertex_elements_state(struct pipe_context * pctx,void * hwcso)597 panfrost_bind_vertex_elements_state(
598 struct pipe_context *pctx,
599 void *hwcso)
600 {
601 struct panfrost_context *ctx = pan_context(pctx);
602 ctx->vertex = hwcso;
603 }
604
605 static void *
panfrost_create_shader_state(struct pipe_context * pctx,const struct pipe_shader_state * cso,enum pipe_shader_type stage)606 panfrost_create_shader_state(
607 struct pipe_context *pctx,
608 const struct pipe_shader_state *cso,
609 enum pipe_shader_type stage)
610 {
611 struct panfrost_shader_variants *so = CALLOC_STRUCT(panfrost_shader_variants);
612 struct panfrost_device *dev = pan_device(pctx->screen);
613 so->base = *cso;
614
615 /* Token deep copy to prevent memory corruption */
616
617 if (cso->type == PIPE_SHADER_IR_TGSI)
618 so->base.tokens = tgsi_dup_tokens(so->base.tokens);
619
620 /* Precompile for shader-db if we need to */
621 if (unlikely((dev->debug & PAN_DBG_PRECOMPILE) && cso->type == PIPE_SHADER_IR_NIR)) {
622 struct panfrost_context *ctx = pan_context(pctx);
623
624 struct panfrost_shader_state state = { 0 };
625 uint64_t outputs_written;
626
627 panfrost_shader_compile(ctx, PIPE_SHADER_IR_NIR,
628 so->base.ir.nir,
629 tgsi_processor_to_shader_stage(stage),
630 &state, &outputs_written);
631 }
632
633 return so;
634 }
635
636 static void
panfrost_delete_shader_state(struct pipe_context * pctx,void * so)637 panfrost_delete_shader_state(
638 struct pipe_context *pctx,
639 void *so)
640 {
641 struct panfrost_shader_variants *cso = (struct panfrost_shader_variants *) so;
642
643 if (cso->base.type == PIPE_SHADER_IR_TGSI) {
644 /* TODO: leaks TGSI tokens! */
645 }
646
647 for (unsigned i = 0; i < cso->variant_count; ++i) {
648 struct panfrost_shader_state *shader_state = &cso->variants[i];
649 panfrost_bo_unreference(shader_state->bo);
650
651 if (shader_state->upload.rsrc)
652 pipe_resource_reference(&shader_state->upload.rsrc, NULL);
653
654 shader_state->bo = NULL;
655 }
656 free(cso->variants);
657
658
659 free(so);
660 }
661
662 static void *
panfrost_create_sampler_state(struct pipe_context * pctx,const struct pipe_sampler_state * cso)663 panfrost_create_sampler_state(
664 struct pipe_context *pctx,
665 const struct pipe_sampler_state *cso)
666 {
667 struct panfrost_sampler_state *so = CALLOC_STRUCT(panfrost_sampler_state);
668 struct panfrost_device *device = pan_device(pctx->screen);
669
670 so->base = *cso;
671
672 if (device->quirks & IS_BIFROST)
673 panfrost_sampler_desc_init_bifrost(cso, (struct mali_bifrost_sampler_packed *) &so->hw);
674 else
675 panfrost_sampler_desc_init(cso, &so->hw);
676
677 return so;
678 }
679
680 static void
panfrost_bind_sampler_states(struct pipe_context * pctx,enum pipe_shader_type shader,unsigned start_slot,unsigned num_sampler,void ** sampler)681 panfrost_bind_sampler_states(
682 struct pipe_context *pctx,
683 enum pipe_shader_type shader,
684 unsigned start_slot, unsigned num_sampler,
685 void **sampler)
686 {
687 assert(start_slot == 0);
688
689 struct panfrost_context *ctx = pan_context(pctx);
690
691 /* XXX: Should upload, not just copy? */
692 ctx->sampler_count[shader] = num_sampler;
693 memcpy(ctx->samplers[shader], sampler, num_sampler * sizeof (void *));
694 }
695
696 static bool
panfrost_variant_matches(struct panfrost_context * ctx,struct panfrost_shader_state * variant,enum pipe_shader_type type)697 panfrost_variant_matches(
698 struct panfrost_context *ctx,
699 struct panfrost_shader_state *variant,
700 enum pipe_shader_type type)
701 {
702 struct panfrost_device *dev = pan_device(ctx->base.screen);
703
704 if (variant->outputs_read) {
705 struct pipe_framebuffer_state *fb = &ctx->pipe_framebuffer;
706
707 unsigned i;
708 BITSET_FOREACH_SET(i, &variant->outputs_read, 8) {
709 enum pipe_format fmt = PIPE_FORMAT_R8G8B8A8_UNORM;
710
711 if ((fb->nr_cbufs > i) && fb->cbufs[i])
712 fmt = fb->cbufs[i]->format;
713
714 const struct util_format_description *desc =
715 util_format_description(fmt);
716
717 if (pan_format_class_load(desc, dev->quirks) == PAN_FORMAT_NATIVE)
718 fmt = PIPE_FORMAT_NONE;
719
720 if (variant->rt_formats[i] != fmt)
721 return false;
722 }
723 }
724
725 /* Otherwise, we're good to go */
726 return true;
727 }
728
729 /**
730 * Fix an uncompiled shader's stream output info, and produce a bitmask
731 * of which VARYING_SLOT_* are captured for stream output.
732 *
733 * Core Gallium stores output->register_index as a "slot" number, where
734 * slots are assigned consecutively to all outputs in info->outputs_written.
735 * This naive packing of outputs doesn't work for us - we too have slots,
736 * but the layout is defined by the VUE map, which we won't have until we
737 * compile a specific shader variant. So, we remap these and simply store
738 * VARYING_SLOT_* in our copy's output->register_index fields.
739 *
740 * We then produce a bitmask of outputs which are used for SO.
741 *
742 * Implementation from iris.
743 */
744
745 static uint64_t
update_so_info(struct pipe_stream_output_info * so_info,uint64_t outputs_written)746 update_so_info(struct pipe_stream_output_info *so_info,
747 uint64_t outputs_written)
748 {
749 uint64_t so_outputs = 0;
750 uint8_t reverse_map[64] = {0};
751 unsigned slot = 0;
752
753 while (outputs_written)
754 reverse_map[slot++] = u_bit_scan64(&outputs_written);
755
756 for (unsigned i = 0; i < so_info->num_outputs; i++) {
757 struct pipe_stream_output *output = &so_info->output[i];
758
759 /* Map Gallium's condensed "slots" back to real VARYING_SLOT_* enums */
760 output->register_index = reverse_map[output->register_index];
761
762 so_outputs |= 1ull << output->register_index;
763 }
764
765 return so_outputs;
766 }
767
768 static void
panfrost_bind_shader_state(struct pipe_context * pctx,void * hwcso,enum pipe_shader_type type)769 panfrost_bind_shader_state(
770 struct pipe_context *pctx,
771 void *hwcso,
772 enum pipe_shader_type type)
773 {
774 struct panfrost_context *ctx = pan_context(pctx);
775 struct panfrost_device *dev = pan_device(ctx->base.screen);
776 ctx->shader[type] = hwcso;
777
778 if (!hwcso) return;
779
780 /* Match the appropriate variant */
781
782 signed variant = -1;
783 struct panfrost_shader_variants *variants = (struct panfrost_shader_variants *) hwcso;
784
785 for (unsigned i = 0; i < variants->variant_count; ++i) {
786 if (panfrost_variant_matches(ctx, &variants->variants[i], type)) {
787 variant = i;
788 break;
789 }
790 }
791
792 if (variant == -1) {
793 /* No variant matched, so create a new one */
794 variant = variants->variant_count++;
795
796 if (variants->variant_count > variants->variant_space) {
797 unsigned old_space = variants->variant_space;
798
799 variants->variant_space *= 2;
800 if (variants->variant_space == 0)
801 variants->variant_space = 1;
802
803 /* Arbitrary limit to stop runaway programs from
804 * creating an unbounded number of shader variants. */
805 assert(variants->variant_space < 1024);
806
807 unsigned msize = sizeof(struct panfrost_shader_state);
808 variants->variants = realloc(variants->variants,
809 variants->variant_space * msize);
810
811 memset(&variants->variants[old_space], 0,
812 (variants->variant_space - old_space) * msize);
813 }
814
815 struct panfrost_shader_state *v =
816 &variants->variants[variant];
817
818 if (type == PIPE_SHADER_FRAGMENT) {
819 struct pipe_framebuffer_state *fb = &ctx->pipe_framebuffer;
820 for (unsigned i = 0; i < fb->nr_cbufs; ++i) {
821 enum pipe_format fmt = PIPE_FORMAT_R8G8B8A8_UNORM;
822
823 if ((fb->nr_cbufs > i) && fb->cbufs[i])
824 fmt = fb->cbufs[i]->format;
825
826 const struct util_format_description *desc =
827 util_format_description(fmt);
828
829 if (pan_format_class_load(desc, dev->quirks) == PAN_FORMAT_NATIVE)
830 fmt = PIPE_FORMAT_NONE;
831
832 v->rt_formats[i] = fmt;
833 }
834 }
835 }
836
837 /* Select this variant */
838 variants->active_variant = variant;
839
840 struct panfrost_shader_state *shader_state = &variants->variants[variant];
841 assert(panfrost_variant_matches(ctx, shader_state, type));
842
843 /* We finally have a variant, so compile it */
844
845 if (!shader_state->compiled) {
846 uint64_t outputs_written = 0;
847
848 panfrost_shader_compile(ctx, variants->base.type,
849 variants->base.type == PIPE_SHADER_IR_NIR ?
850 variants->base.ir.nir :
851 variants->base.tokens,
852 tgsi_processor_to_shader_stage(type),
853 shader_state,
854 &outputs_written);
855
856 shader_state->compiled = true;
857
858 /* Fixup the stream out information, since what Gallium returns
859 * normally is mildly insane */
860
861 shader_state->stream_output = variants->base.stream_output;
862 shader_state->so_mask =
863 update_so_info(&shader_state->stream_output, outputs_written);
864 }
865 }
866
867 static void *
panfrost_create_vs_state(struct pipe_context * pctx,const struct pipe_shader_state * hwcso)868 panfrost_create_vs_state(struct pipe_context *pctx, const struct pipe_shader_state *hwcso)
869 {
870 return panfrost_create_shader_state(pctx, hwcso, PIPE_SHADER_VERTEX);
871 }
872
873 static void *
panfrost_create_fs_state(struct pipe_context * pctx,const struct pipe_shader_state * hwcso)874 panfrost_create_fs_state(struct pipe_context *pctx, const struct pipe_shader_state *hwcso)
875 {
876 return panfrost_create_shader_state(pctx, hwcso, PIPE_SHADER_FRAGMENT);
877 }
878
879 static void
panfrost_bind_vs_state(struct pipe_context * pctx,void * hwcso)880 panfrost_bind_vs_state(struct pipe_context *pctx, void *hwcso)
881 {
882 panfrost_bind_shader_state(pctx, hwcso, PIPE_SHADER_VERTEX);
883 }
884
885 static void
panfrost_bind_fs_state(struct pipe_context * pctx,void * hwcso)886 panfrost_bind_fs_state(struct pipe_context *pctx, void *hwcso)
887 {
888 panfrost_bind_shader_state(pctx, hwcso, PIPE_SHADER_FRAGMENT);
889 }
890
891 static void
panfrost_set_vertex_buffers(struct pipe_context * pctx,unsigned start_slot,unsigned num_buffers,const struct pipe_vertex_buffer * buffers)892 panfrost_set_vertex_buffers(
893 struct pipe_context *pctx,
894 unsigned start_slot,
895 unsigned num_buffers,
896 const struct pipe_vertex_buffer *buffers)
897 {
898 struct panfrost_context *ctx = pan_context(pctx);
899
900 util_set_vertex_buffers_mask(ctx->vertex_buffers, &ctx->vb_mask, buffers, start_slot, num_buffers);
901 }
902
903 static void
panfrost_set_constant_buffer(struct pipe_context * pctx,enum pipe_shader_type shader,uint index,const struct pipe_constant_buffer * buf)904 panfrost_set_constant_buffer(
905 struct pipe_context *pctx,
906 enum pipe_shader_type shader, uint index,
907 const struct pipe_constant_buffer *buf)
908 {
909 struct panfrost_context *ctx = pan_context(pctx);
910 struct panfrost_constant_buffer *pbuf = &ctx->constant_buffer[shader];
911
912 util_copy_constant_buffer(&pbuf->cb[index], buf);
913
914 unsigned mask = (1 << index);
915
916 if (unlikely(!buf)) {
917 pbuf->enabled_mask &= ~mask;
918 pbuf->dirty_mask &= ~mask;
919 return;
920 }
921
922 pbuf->enabled_mask |= mask;
923 pbuf->dirty_mask |= mask;
924 }
925
926 static void
panfrost_set_stencil_ref(struct pipe_context * pctx,const struct pipe_stencil_ref * ref)927 panfrost_set_stencil_ref(
928 struct pipe_context *pctx,
929 const struct pipe_stencil_ref *ref)
930 {
931 struct panfrost_context *ctx = pan_context(pctx);
932 ctx->stencil_ref = *ref;
933 }
934
935 void
panfrost_create_sampler_view_bo(struct panfrost_sampler_view * so,struct pipe_context * pctx,struct pipe_resource * texture)936 panfrost_create_sampler_view_bo(struct panfrost_sampler_view *so,
937 struct pipe_context *pctx,
938 struct pipe_resource *texture)
939 {
940 struct panfrost_device *device = pan_device(pctx->screen);
941 struct panfrost_resource *prsrc = (struct panfrost_resource *)texture;
942 enum pipe_format format = so->base.format;
943 assert(prsrc->bo);
944
945 /* Format to access the stencil portion of a Z32_S8 texture */
946 if (format == PIPE_FORMAT_X32_S8X24_UINT) {
947 assert(prsrc->separate_stencil);
948 texture = &prsrc->separate_stencil->base;
949 prsrc = (struct panfrost_resource *)texture;
950 format = texture->format;
951 }
952
953 const struct util_format_description *desc = util_format_description(format);
954
955 bool fake_rgtc = !panfrost_supports_compressed_format(device, MALI_BC4_UNORM);
956
957 if (desc->layout == UTIL_FORMAT_LAYOUT_RGTC && fake_rgtc) {
958 if (desc->is_snorm)
959 format = PIPE_FORMAT_R8G8B8A8_SNORM;
960 else
961 format = PIPE_FORMAT_R8G8B8A8_UNORM;
962 desc = util_format_description(format);
963 }
964
965 so->texture_bo = prsrc->bo->ptr.gpu;
966 so->modifier = prsrc->modifier;
967
968 unsigned char user_swizzle[4] = {
969 so->base.swizzle_r,
970 so->base.swizzle_g,
971 so->base.swizzle_b,
972 so->base.swizzle_a
973 };
974
975 /* In the hardware, array_size refers specifically to array textures,
976 * whereas in Gallium, it also covers cubemaps */
977
978 unsigned array_size = texture->array_size;
979 unsigned depth = texture->depth0;
980
981 if (so->base.target == PIPE_TEXTURE_CUBE) {
982 /* TODO: Cubemap arrays */
983 assert(array_size == 6);
984 array_size /= 6;
985 }
986
987 /* MSAA only supported for 2D textures (and 2D texture arrays via an
988 * extension currently unimplemented */
989
990 if (so->base.target == PIPE_TEXTURE_2D) {
991 assert(depth == 1);
992 depth = texture->nr_samples;
993 } else {
994 /* MSAA only supported for 2D textures */
995 assert(texture->nr_samples <= 1);
996 }
997
998 enum mali_texture_dimension type =
999 panfrost_translate_texture_dimension(so->base.target);
1000
1001 if (device->quirks & IS_BIFROST) {
1002 unsigned size = panfrost_estimate_texture_payload_size(
1003 so->base.u.tex.first_level,
1004 so->base.u.tex.last_level,
1005 so->base.u.tex.first_layer,
1006 so->base.u.tex.last_layer,
1007 texture->nr_samples,
1008 type, prsrc->modifier);
1009
1010 so->bo = panfrost_bo_create(device, size, 0);
1011
1012 panfrost_new_texture_bifrost(
1013 device,
1014 &so->bifrost_descriptor,
1015 texture->width0, texture->height0,
1016 depth, array_size,
1017 format,
1018 type, prsrc->modifier,
1019 so->base.u.tex.first_level,
1020 so->base.u.tex.last_level,
1021 so->base.u.tex.first_layer,
1022 so->base.u.tex.last_layer,
1023 texture->nr_samples,
1024 prsrc->cubemap_stride,
1025 panfrost_translate_swizzle_4(user_swizzle),
1026 prsrc->bo->ptr.gpu,
1027 prsrc->slices, &so->bo->ptr);
1028 } else {
1029 unsigned size = panfrost_estimate_texture_payload_size(
1030 so->base.u.tex.first_level,
1031 so->base.u.tex.last_level,
1032 so->base.u.tex.first_layer,
1033 so->base.u.tex.last_layer,
1034 texture->nr_samples,
1035 type, prsrc->modifier);
1036 size += MALI_MIDGARD_TEXTURE_LENGTH;
1037
1038 so->bo = panfrost_bo_create(device, size, 0);
1039
1040 panfrost_new_texture(
1041 so->bo->ptr.cpu,
1042 texture->width0, texture->height0,
1043 depth, array_size,
1044 format,
1045 type, prsrc->modifier,
1046 so->base.u.tex.first_level,
1047 so->base.u.tex.last_level,
1048 so->base.u.tex.first_layer,
1049 so->base.u.tex.last_layer,
1050 texture->nr_samples,
1051 prsrc->cubemap_stride,
1052 panfrost_translate_swizzle_4(user_swizzle),
1053 prsrc->bo->ptr.gpu,
1054 prsrc->slices);
1055 }
1056 }
1057
1058 static struct pipe_sampler_view *
panfrost_create_sampler_view(struct pipe_context * pctx,struct pipe_resource * texture,const struct pipe_sampler_view * template)1059 panfrost_create_sampler_view(
1060 struct pipe_context *pctx,
1061 struct pipe_resource *texture,
1062 const struct pipe_sampler_view *template)
1063 {
1064 struct panfrost_sampler_view *so = rzalloc(pctx, struct panfrost_sampler_view);
1065
1066 pipe_reference(NULL, &texture->reference);
1067
1068 so->base = *template;
1069 so->base.texture = texture;
1070 so->base.reference.count = 1;
1071 so->base.context = pctx;
1072
1073 panfrost_create_sampler_view_bo(so, pctx, texture);
1074
1075 return (struct pipe_sampler_view *) so;
1076 }
1077
1078 static void
panfrost_set_sampler_views(struct pipe_context * pctx,enum pipe_shader_type shader,unsigned start_slot,unsigned num_views,struct pipe_sampler_view ** views)1079 panfrost_set_sampler_views(
1080 struct pipe_context *pctx,
1081 enum pipe_shader_type shader,
1082 unsigned start_slot, unsigned num_views,
1083 struct pipe_sampler_view **views)
1084 {
1085 struct panfrost_context *ctx = pan_context(pctx);
1086 unsigned new_nr = 0;
1087 unsigned i;
1088
1089 assert(start_slot == 0);
1090
1091 for (i = 0; i < num_views; ++i) {
1092 if (views[i])
1093 new_nr = i + 1;
1094 pipe_sampler_view_reference((struct pipe_sampler_view **)&ctx->sampler_views[shader][i],
1095 views[i]);
1096 }
1097
1098 for (; i < ctx->sampler_view_count[shader]; i++) {
1099 pipe_sampler_view_reference((struct pipe_sampler_view **)&ctx->sampler_views[shader][i],
1100 NULL);
1101 }
1102 ctx->sampler_view_count[shader] = new_nr;
1103 }
1104
1105 static void
panfrost_sampler_view_destroy(struct pipe_context * pctx,struct pipe_sampler_view * pview)1106 panfrost_sampler_view_destroy(
1107 struct pipe_context *pctx,
1108 struct pipe_sampler_view *pview)
1109 {
1110 struct panfrost_sampler_view *view = (struct panfrost_sampler_view *) pview;
1111
1112 pipe_resource_reference(&pview->texture, NULL);
1113 panfrost_bo_unreference(view->bo);
1114 ralloc_free(view);
1115 }
1116
1117 static void
panfrost_set_shader_buffers(struct pipe_context * pctx,enum pipe_shader_type shader,unsigned start,unsigned count,const struct pipe_shader_buffer * buffers,unsigned writable_bitmask)1118 panfrost_set_shader_buffers(
1119 struct pipe_context *pctx,
1120 enum pipe_shader_type shader,
1121 unsigned start, unsigned count,
1122 const struct pipe_shader_buffer *buffers,
1123 unsigned writable_bitmask)
1124 {
1125 struct panfrost_context *ctx = pan_context(pctx);
1126
1127 util_set_shader_buffers_mask(ctx->ssbo[shader], &ctx->ssbo_mask[shader],
1128 buffers, start, count);
1129 }
1130
1131 static void
panfrost_set_shader_images(struct pipe_context * pctx,enum pipe_shader_type shader,unsigned start,unsigned count,const struct pipe_image_view * images)1132 panfrost_set_shader_images(
1133 struct pipe_context *pctx,
1134 enum pipe_shader_type shader,
1135 unsigned start, unsigned count,
1136 const struct pipe_image_view *images)
1137 {
1138 /* TODO */
1139 }
1140
1141 static void
panfrost_set_framebuffer_state(struct pipe_context * pctx,const struct pipe_framebuffer_state * fb)1142 panfrost_set_framebuffer_state(struct pipe_context *pctx,
1143 const struct pipe_framebuffer_state *fb)
1144 {
1145 struct panfrost_context *ctx = pan_context(pctx);
1146
1147 util_copy_framebuffer_state(&ctx->pipe_framebuffer, fb);
1148 ctx->batch = NULL;
1149
1150 /* We may need to generate a new variant if the fragment shader is
1151 * keyed to the framebuffer format (due to EXT_framebuffer_fetch) */
1152 struct panfrost_shader_variants *fs = ctx->shader[PIPE_SHADER_FRAGMENT];
1153
1154 if (fs && fs->variant_count && fs->variants[fs->active_variant].outputs_read)
1155 ctx->base.bind_fs_state(&ctx->base, fs);
1156 }
1157
1158 static inline unsigned
pan_pipe_to_stencil_op(enum pipe_stencil_op in)1159 pan_pipe_to_stencil_op(enum pipe_stencil_op in)
1160 {
1161 switch (in) {
1162 case PIPE_STENCIL_OP_KEEP: return MALI_STENCIL_OP_KEEP;
1163 case PIPE_STENCIL_OP_ZERO: return MALI_STENCIL_OP_ZERO;
1164 case PIPE_STENCIL_OP_REPLACE: return MALI_STENCIL_OP_REPLACE;
1165 case PIPE_STENCIL_OP_INCR: return MALI_STENCIL_OP_INCR_SAT;
1166 case PIPE_STENCIL_OP_DECR: return MALI_STENCIL_OP_DECR_SAT;
1167 case PIPE_STENCIL_OP_INCR_WRAP: return MALI_STENCIL_OP_INCR_WRAP;
1168 case PIPE_STENCIL_OP_DECR_WRAP: return MALI_STENCIL_OP_DECR_WRAP;
1169 case PIPE_STENCIL_OP_INVERT: return MALI_STENCIL_OP_INVERT;
1170 default: unreachable("Invalid stencil op");
1171 }
1172 }
1173
1174 static inline void
pan_pipe_to_stencil(const struct pipe_stencil_state * in,struct MALI_STENCIL * out)1175 pan_pipe_to_stencil(const struct pipe_stencil_state *in, struct MALI_STENCIL *out)
1176 {
1177 pan_prepare(out, STENCIL);
1178 out->mask = in->valuemask;
1179 out->compare_function = panfrost_translate_compare_func(in->func);
1180 out->stencil_fail = pan_pipe_to_stencil_op(in->fail_op);
1181 out->depth_fail = pan_pipe_to_stencil_op(in->zfail_op);
1182 out->depth_pass = pan_pipe_to_stencil_op(in->zpass_op);
1183 }
1184
1185 static void *
panfrost_create_depth_stencil_state(struct pipe_context * pipe,const struct pipe_depth_stencil_alpha_state * zsa)1186 panfrost_create_depth_stencil_state(struct pipe_context *pipe,
1187 const struct pipe_depth_stencil_alpha_state *zsa)
1188 {
1189 struct panfrost_zsa_state *so = CALLOC_STRUCT(panfrost_zsa_state);
1190 so->base = *zsa;
1191
1192 pan_pipe_to_stencil(&zsa->stencil[0], &so->stencil_front);
1193 so->stencil_mask_front = zsa->stencil[0].writemask;
1194
1195 if (zsa->stencil[1].enabled) {
1196 pan_pipe_to_stencil(&zsa->stencil[1], &so->stencil_back);
1197 so->stencil_mask_back = zsa->stencil[1].writemask;
1198 } else {
1199 so->stencil_back = so->stencil_front;
1200 so->stencil_mask_back = so->stencil_mask_front;
1201 }
1202
1203 /* Alpha lowered by frontend */
1204 assert(!zsa->alpha.enabled);
1205
1206 /* TODO: Bounds test should be easy */
1207 assert(!zsa->depth.bounds_test);
1208
1209 return so;
1210 }
1211
1212 static void
panfrost_bind_depth_stencil_state(struct pipe_context * pipe,void * cso)1213 panfrost_bind_depth_stencil_state(struct pipe_context *pipe,
1214 void *cso)
1215 {
1216 struct panfrost_context *ctx = pan_context(pipe);
1217 ctx->depth_stencil = cso;
1218 }
1219
1220 static void
panfrost_delete_depth_stencil_state(struct pipe_context * pipe,void * depth)1221 panfrost_delete_depth_stencil_state(struct pipe_context *pipe, void *depth)
1222 {
1223 free( depth );
1224 }
1225
1226 static void
panfrost_set_sample_mask(struct pipe_context * pipe,unsigned sample_mask)1227 panfrost_set_sample_mask(struct pipe_context *pipe,
1228 unsigned sample_mask)
1229 {
1230 struct panfrost_context *ctx = pan_context(pipe);
1231 ctx->sample_mask = sample_mask;
1232 }
1233
1234 static void
panfrost_set_min_samples(struct pipe_context * pipe,unsigned min_samples)1235 panfrost_set_min_samples(struct pipe_context *pipe,
1236 unsigned min_samples)
1237 {
1238 struct panfrost_context *ctx = pan_context(pipe);
1239 ctx->min_samples = min_samples;
1240 }
1241
1242
1243 static void
panfrost_set_clip_state(struct pipe_context * pipe,const struct pipe_clip_state * clip)1244 panfrost_set_clip_state(struct pipe_context *pipe,
1245 const struct pipe_clip_state *clip)
1246 {
1247 //struct panfrost_context *panfrost = pan_context(pipe);
1248 }
1249
1250 static void
panfrost_set_viewport_states(struct pipe_context * pipe,unsigned start_slot,unsigned num_viewports,const struct pipe_viewport_state * viewports)1251 panfrost_set_viewport_states(struct pipe_context *pipe,
1252 unsigned start_slot,
1253 unsigned num_viewports,
1254 const struct pipe_viewport_state *viewports)
1255 {
1256 struct panfrost_context *ctx = pan_context(pipe);
1257
1258 assert(start_slot == 0);
1259 assert(num_viewports == 1);
1260
1261 ctx->pipe_viewport = *viewports;
1262 }
1263
1264 static void
panfrost_set_scissor_states(struct pipe_context * pipe,unsigned start_slot,unsigned num_scissors,const struct pipe_scissor_state * scissors)1265 panfrost_set_scissor_states(struct pipe_context *pipe,
1266 unsigned start_slot,
1267 unsigned num_scissors,
1268 const struct pipe_scissor_state *scissors)
1269 {
1270 struct panfrost_context *ctx = pan_context(pipe);
1271
1272 assert(start_slot == 0);
1273 assert(num_scissors == 1);
1274
1275 ctx->scissor = *scissors;
1276 }
1277
1278 static void
panfrost_set_polygon_stipple(struct pipe_context * pipe,const struct pipe_poly_stipple * stipple)1279 panfrost_set_polygon_stipple(struct pipe_context *pipe,
1280 const struct pipe_poly_stipple *stipple)
1281 {
1282 //struct panfrost_context *panfrost = pan_context(pipe);
1283 }
1284
1285 static void
panfrost_set_active_query_state(struct pipe_context * pipe,bool enable)1286 panfrost_set_active_query_state(struct pipe_context *pipe,
1287 bool enable)
1288 {
1289 struct panfrost_context *ctx = pan_context(pipe);
1290 ctx->active_queries = enable;
1291 }
1292
1293 static void
panfrost_destroy(struct pipe_context * pipe)1294 panfrost_destroy(struct pipe_context *pipe)
1295 {
1296 struct panfrost_context *panfrost = pan_context(pipe);
1297
1298 if (panfrost->blitter)
1299 util_blitter_destroy(panfrost->blitter);
1300
1301 util_unreference_framebuffer_state(&panfrost->pipe_framebuffer);
1302 u_upload_destroy(pipe->stream_uploader);
1303 u_upload_destroy(panfrost->state_uploader);
1304
1305 ralloc_free(pipe);
1306 }
1307
1308 static struct pipe_query *
panfrost_create_query(struct pipe_context * pipe,unsigned type,unsigned index)1309 panfrost_create_query(struct pipe_context *pipe,
1310 unsigned type,
1311 unsigned index)
1312 {
1313 struct panfrost_query *q = rzalloc(pipe, struct panfrost_query);
1314
1315 q->type = type;
1316 q->index = index;
1317
1318 return (struct pipe_query *) q;
1319 }
1320
1321 static void
panfrost_destroy_query(struct pipe_context * pipe,struct pipe_query * q)1322 panfrost_destroy_query(struct pipe_context *pipe, struct pipe_query *q)
1323 {
1324 struct panfrost_query *query = (struct panfrost_query *) q;
1325
1326 if (query->bo) {
1327 panfrost_bo_unreference(query->bo);
1328 query->bo = NULL;
1329 }
1330
1331 ralloc_free(q);
1332 }
1333
1334 static bool
panfrost_begin_query(struct pipe_context * pipe,struct pipe_query * q)1335 panfrost_begin_query(struct pipe_context *pipe, struct pipe_query *q)
1336 {
1337 struct panfrost_context *ctx = pan_context(pipe);
1338 struct panfrost_device *dev = pan_device(ctx->base.screen);
1339 struct panfrost_query *query = (struct panfrost_query *) q;
1340
1341 switch (query->type) {
1342 case PIPE_QUERY_OCCLUSION_COUNTER:
1343 case PIPE_QUERY_OCCLUSION_PREDICATE:
1344 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE: {
1345 unsigned size = sizeof(uint64_t) * dev->core_count;
1346
1347 /* Allocate a bo for the query results to be stored */
1348 if (!query->bo) {
1349 query->bo = panfrost_bo_create(dev, size, 0);
1350 }
1351
1352 /* Default to 0 if nothing at all drawn. */
1353 memset(query->bo->ptr.cpu, 0, size);
1354
1355 query->msaa = (ctx->pipe_framebuffer.samples > 1);
1356 ctx->occlusion_query = query;
1357 break;
1358 }
1359
1360 /* Geometry statistics are computed in the driver. XXX: geom/tess
1361 * shaders.. */
1362
1363 case PIPE_QUERY_PRIMITIVES_GENERATED:
1364 query->start = ctx->prims_generated;
1365 break;
1366 case PIPE_QUERY_PRIMITIVES_EMITTED:
1367 query->start = ctx->tf_prims_generated;
1368 break;
1369
1370 default:
1371 /* TODO: timestamp queries, etc? */
1372 break;
1373 }
1374
1375 return true;
1376 }
1377
1378 static bool
panfrost_end_query(struct pipe_context * pipe,struct pipe_query * q)1379 panfrost_end_query(struct pipe_context *pipe, struct pipe_query *q)
1380 {
1381 struct panfrost_context *ctx = pan_context(pipe);
1382 struct panfrost_query *query = (struct panfrost_query *) q;
1383
1384 switch (query->type) {
1385 case PIPE_QUERY_OCCLUSION_COUNTER:
1386 case PIPE_QUERY_OCCLUSION_PREDICATE:
1387 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
1388 ctx->occlusion_query = NULL;
1389 break;
1390 case PIPE_QUERY_PRIMITIVES_GENERATED:
1391 query->end = ctx->prims_generated;
1392 break;
1393 case PIPE_QUERY_PRIMITIVES_EMITTED:
1394 query->end = ctx->tf_prims_generated;
1395 break;
1396 }
1397
1398 return true;
1399 }
1400
1401 static bool
panfrost_get_query_result(struct pipe_context * pipe,struct pipe_query * q,bool wait,union pipe_query_result * vresult)1402 panfrost_get_query_result(struct pipe_context *pipe,
1403 struct pipe_query *q,
1404 bool wait,
1405 union pipe_query_result *vresult)
1406 {
1407 struct panfrost_query *query = (struct panfrost_query *) q;
1408 struct panfrost_context *ctx = pan_context(pipe);
1409 struct panfrost_device *dev = pan_device(ctx->base.screen);
1410
1411 switch (query->type) {
1412 case PIPE_QUERY_OCCLUSION_COUNTER:
1413 case PIPE_QUERY_OCCLUSION_PREDICATE:
1414 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
1415 panfrost_flush_batches_accessing_bo(ctx, query->bo, false);
1416 panfrost_bo_wait(query->bo, INT64_MAX, false);
1417
1418 /* Read back the query results */
1419 uint64_t *result = (uint64_t *) query->bo->ptr.cpu;
1420
1421 if (query->type == PIPE_QUERY_OCCLUSION_COUNTER) {
1422 uint64_t passed = 0;
1423 for (int i = 0; i < dev->core_count; ++i)
1424 passed += result[i];
1425
1426 if (!query->msaa)
1427 passed /= 4;
1428
1429 vresult->u64 = passed;
1430 } else {
1431 vresult->b = !!result[0];
1432 }
1433
1434 break;
1435
1436 case PIPE_QUERY_PRIMITIVES_GENERATED:
1437 case PIPE_QUERY_PRIMITIVES_EMITTED:
1438 panfrost_flush_all_batches(ctx);
1439 vresult->u64 = query->end - query->start;
1440 break;
1441
1442 default:
1443 /* TODO: more queries */
1444 break;
1445 }
1446
1447 return true;
1448 }
1449
1450 static struct pipe_stream_output_target *
panfrost_create_stream_output_target(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned buffer_offset,unsigned buffer_size)1451 panfrost_create_stream_output_target(struct pipe_context *pctx,
1452 struct pipe_resource *prsc,
1453 unsigned buffer_offset,
1454 unsigned buffer_size)
1455 {
1456 struct pipe_stream_output_target *target;
1457
1458 target = &rzalloc(pctx, struct panfrost_streamout_target)->base;
1459
1460 if (!target)
1461 return NULL;
1462
1463 pipe_reference_init(&target->reference, 1);
1464 pipe_resource_reference(&target->buffer, prsc);
1465
1466 target->context = pctx;
1467 target->buffer_offset = buffer_offset;
1468 target->buffer_size = buffer_size;
1469
1470 return target;
1471 }
1472
1473 static void
panfrost_stream_output_target_destroy(struct pipe_context * pctx,struct pipe_stream_output_target * target)1474 panfrost_stream_output_target_destroy(struct pipe_context *pctx,
1475 struct pipe_stream_output_target *target)
1476 {
1477 pipe_resource_reference(&target->buffer, NULL);
1478 ralloc_free(target);
1479 }
1480
1481 static void
panfrost_set_stream_output_targets(struct pipe_context * pctx,unsigned num_targets,struct pipe_stream_output_target ** targets,const unsigned * offsets)1482 panfrost_set_stream_output_targets(struct pipe_context *pctx,
1483 unsigned num_targets,
1484 struct pipe_stream_output_target **targets,
1485 const unsigned *offsets)
1486 {
1487 struct panfrost_context *ctx = pan_context(pctx);
1488 struct panfrost_streamout *so = &ctx->streamout;
1489
1490 assert(num_targets <= ARRAY_SIZE(so->targets));
1491
1492 for (unsigned i = 0; i < num_targets; i++) {
1493 if (offsets[i] != -1)
1494 pan_so_target(targets[i])->offset = offsets[i];
1495
1496 pipe_so_target_reference(&so->targets[i], targets[i]);
1497 }
1498
1499 for (unsigned i = 0; i < so->num_targets; i++)
1500 pipe_so_target_reference(&so->targets[i], NULL);
1501
1502 so->num_targets = num_targets;
1503 }
1504
panfrost_shader_key_hash(const void * key)1505 static uint32_t panfrost_shader_key_hash(const void *key)
1506 {
1507 return _mesa_hash_data(key, sizeof(struct panfrost_blend_shader_key));
1508 }
1509
panfrost_shader_key_equal(const void * a,const void * b)1510 static bool panfrost_shader_key_equal(const void *a, const void *b)
1511 {
1512 return !memcmp(a, b, sizeof(struct panfrost_blend_shader_key));
1513 }
1514
1515 struct pipe_context *
panfrost_create_context(struct pipe_screen * screen,void * priv,unsigned flags)1516 panfrost_create_context(struct pipe_screen *screen, void *priv, unsigned flags)
1517 {
1518 struct panfrost_context *ctx = rzalloc(screen, struct panfrost_context);
1519 struct pipe_context *gallium = (struct pipe_context *) ctx;
1520 struct panfrost_device *dev = pan_device(screen);
1521
1522 gallium->screen = screen;
1523
1524 gallium->destroy = panfrost_destroy;
1525
1526 gallium->set_framebuffer_state = panfrost_set_framebuffer_state;
1527
1528 gallium->flush = panfrost_flush;
1529 gallium->clear = panfrost_clear;
1530 gallium->draw_vbo = panfrost_draw_vbo;
1531 gallium->texture_barrier = panfrost_texture_barrier;
1532
1533 gallium->set_vertex_buffers = panfrost_set_vertex_buffers;
1534 gallium->set_constant_buffer = panfrost_set_constant_buffer;
1535 gallium->set_shader_buffers = panfrost_set_shader_buffers;
1536 gallium->set_shader_images = panfrost_set_shader_images;
1537
1538 gallium->set_stencil_ref = panfrost_set_stencil_ref;
1539
1540 gallium->create_sampler_view = panfrost_create_sampler_view;
1541 gallium->set_sampler_views = panfrost_set_sampler_views;
1542 gallium->sampler_view_destroy = panfrost_sampler_view_destroy;
1543
1544 gallium->create_rasterizer_state = panfrost_create_rasterizer_state;
1545 gallium->bind_rasterizer_state = panfrost_bind_rasterizer_state;
1546 gallium->delete_rasterizer_state = panfrost_generic_cso_delete;
1547
1548 gallium->create_vertex_elements_state = panfrost_create_vertex_elements_state;
1549 gallium->bind_vertex_elements_state = panfrost_bind_vertex_elements_state;
1550 gallium->delete_vertex_elements_state = panfrost_generic_cso_delete;
1551
1552 gallium->create_fs_state = panfrost_create_fs_state;
1553 gallium->delete_fs_state = panfrost_delete_shader_state;
1554 gallium->bind_fs_state = panfrost_bind_fs_state;
1555
1556 gallium->create_vs_state = panfrost_create_vs_state;
1557 gallium->delete_vs_state = panfrost_delete_shader_state;
1558 gallium->bind_vs_state = panfrost_bind_vs_state;
1559
1560 gallium->create_sampler_state = panfrost_create_sampler_state;
1561 gallium->delete_sampler_state = panfrost_generic_cso_delete;
1562 gallium->bind_sampler_states = panfrost_bind_sampler_states;
1563
1564 gallium->create_depth_stencil_alpha_state = panfrost_create_depth_stencil_state;
1565 gallium->bind_depth_stencil_alpha_state = panfrost_bind_depth_stencil_state;
1566 gallium->delete_depth_stencil_alpha_state = panfrost_delete_depth_stencil_state;
1567
1568 gallium->set_sample_mask = panfrost_set_sample_mask;
1569 gallium->set_min_samples = panfrost_set_min_samples;
1570
1571 gallium->set_clip_state = panfrost_set_clip_state;
1572 gallium->set_viewport_states = panfrost_set_viewport_states;
1573 gallium->set_scissor_states = panfrost_set_scissor_states;
1574 gallium->set_polygon_stipple = panfrost_set_polygon_stipple;
1575 gallium->set_active_query_state = panfrost_set_active_query_state;
1576
1577 gallium->create_query = panfrost_create_query;
1578 gallium->destroy_query = panfrost_destroy_query;
1579 gallium->begin_query = panfrost_begin_query;
1580 gallium->end_query = panfrost_end_query;
1581 gallium->get_query_result = panfrost_get_query_result;
1582
1583 gallium->create_stream_output_target = panfrost_create_stream_output_target;
1584 gallium->stream_output_target_destroy = panfrost_stream_output_target_destroy;
1585 gallium->set_stream_output_targets = panfrost_set_stream_output_targets;
1586
1587 panfrost_resource_context_init(gallium);
1588 panfrost_blend_context_init(gallium);
1589 panfrost_compute_context_init(gallium);
1590
1591 gallium->stream_uploader = u_upload_create_default(gallium);
1592 gallium->const_uploader = gallium->stream_uploader;
1593
1594 ctx->state_uploader = u_upload_create(gallium, 4096,
1595 PIPE_BIND_CONSTANT_BUFFER, PIPE_USAGE_DYNAMIC, 0);
1596
1597 /* All of our GPUs support ES mode. Midgard supports additionally
1598 * QUADS/QUAD_STRIPS/POLYGON. Bifrost supports just QUADS. */
1599
1600 ctx->draw_modes = (1 << (PIPE_PRIM_QUADS + 1)) - 1;
1601
1602 if (!(dev->quirks & IS_BIFROST)) {
1603 ctx->draw_modes |= (1 << PIPE_PRIM_QUAD_STRIP);
1604 ctx->draw_modes |= (1 << PIPE_PRIM_POLYGON);
1605 }
1606
1607 ctx->primconvert = util_primconvert_create(gallium, ctx->draw_modes);
1608
1609 ctx->blitter = util_blitter_create(gallium);
1610
1611 assert(ctx->blitter);
1612
1613 /* Prepare for render! */
1614
1615 panfrost_batch_init(ctx);
1616
1617 ctx->blit_blend = rzalloc(ctx, struct panfrost_blend_state);
1618 ctx->blend_shaders =
1619 _mesa_hash_table_create(ctx,
1620 panfrost_shader_key_hash,
1621 panfrost_shader_key_equal);
1622
1623 /* By default mask everything on */
1624 ctx->sample_mask = ~0;
1625 ctx->active_queries = true;
1626
1627 int ASSERTED ret;
1628
1629 /* Create a syncobj in a signaled state. Will be updated to point to the
1630 * last queued job out_sync every time we submit a new job.
1631 */
1632 ret = drmSyncobjCreate(dev->fd, DRM_SYNCOBJ_CREATE_SIGNALED, &ctx->syncobj);
1633 assert(!ret && ctx->syncobj);
1634
1635 return gallium;
1636 }
1637