45 	struct si_context *sctx = (struct si_context *)ctx;  in si_blitter_begin()  local
47 	util_blitter_save_vertex_shader(sctx->blitter, sctx->vs_shader.cso);  in si_blitter_begin()
48 	util_blitter_save_tessctrl_shader(sctx->blitter, sctx->tcs_shader.cso);  in si_blitter_begin()
49 	util_blitter_save_tesseval_shader(sctx->blitter, sctx->tes_shader.cso);  in si_blitter_begin()
50 	util_blitter_save_geometry_shader(sctx->blitter, sctx->gs_shader.cso);  in si_blitter_begin()
51 	util_blitter_save_so_targets(sctx->blitter, sctx->streamout.num_targets,  in si_blitter_begin()
52 				     (struct pipe_stream_output_target**)sctx->streamout.targets);  in si_blitter_begin()
53 	util_blitter_save_rasterizer(sctx->blitter, sctx->queued.named.rasterizer);  in si_blitter_begin()
56 		util_blitter_save_blend(sctx->blitter, sctx->queued.named.blend);  in si_blitter_begin()
57 		util_blitter_save_depth_stencil_alpha(sctx->blitter, sctx->queued.named.dsa);  in si_blitter_begin()
58 		util_blitter_save_stencil_ref(sctx->blitter, &sctx->stencil_ref.state);  in si_blitter_begin()
59 		util_blitter_save_fragment_shader(sctx->blitter, sctx->ps_shader.cso);  in si_blitter_begin()
60 		util_blitter_save_sample_mask(sctx->blitter, sctx->sample_mask.sample_mask);  in si_blitter_begin()
61 		util_blitter_save_scissor(sctx->blitter, &sctx->scissors.states[0]);  in si_blitter_begin()
65 		util_blitter_save_framebuffer(sctx->blitter, &sctx->framebuffer.state);  in si_blitter_begin()
69 			sctx->blitter, 2,  in si_blitter_begin()
70 			(void**)sctx->samplers[PIPE_SHADER_FRAGMENT].sampler_states);  in si_blitter_begin()
72 		util_blitter_save_fragment_sampler_views(sctx->blitter, 2,  in si_blitter_begin()
73 			sctx->samplers[PIPE_SHADER_FRAGMENT].views);  in si_blitter_begin()
77 		sctx->b.render_cond_force_off = true;  in si_blitter_begin()
82 	struct si_context *sctx = (struct si_context *)ctx;  in si_blitter_end()  local
84 	sctx->b.render_cond_force_off = false;  in si_blitter_end()
88 	sctx->shader_pointers_dirty |= SI_DESCS_SHADER_MASK(VERTEX);  in si_blitter_end()
89 	sctx->vertex_buffer_pointer_dirty = true;  in si_blitter_end()
90 	si_mark_atom_dirty(sctx, &sctx->shader_pointers.atom);  in si_blitter_end()
99 si_blit_dbcb_copy(struct si_context *sctx,  in si_blit_dbcb_copy()  argument
111 		sctx->dbcb_depth_copy_enabled = true;  in si_blit_dbcb_copy()
113 		sctx->dbcb_stencil_copy_enabled = true;  in si_blit_dbcb_copy()
114 	si_mark_atom_dirty(sctx, &sctx->db_render_state);  in si_blit_dbcb_copy()
116 	assert(sctx->dbcb_depth_copy_enabled || sctx->dbcb_stencil_copy_enabled);  in si_blit_dbcb_copy()
118 	sctx->decompression_enabled = true;  in si_blit_dbcb_copy()
137 			zsurf = sctx->b.b.create_surface(&sctx->b.b, &src->resource.b.b, &surf_tmpl);  in si_blit_dbcb_copy()
140 			cbsurf = sctx->b.b.create_surface(&sctx->b.b, &dst->resource.b.b, &surf_tmpl);  in si_blit_dbcb_copy()
143 				if (sample != sctx->dbcb_copy_sample) {  in si_blit_dbcb_copy()
144 					sctx->dbcb_copy_sample = sample;  in si_blit_dbcb_copy()
145 					si_mark_atom_dirty(sctx, &sctx->db_render_state);  in si_blit_dbcb_copy()
148 				si_blitter_begin(&sctx->b.b, SI_DECOMPRESS);  in si_blit_dbcb_copy()
149 				util_blitter_custom_depth_stencil(sctx->blitter, zsurf, cbsurf, 1 << sample,  in si_blit_dbcb_copy()
150 								  sctx->custom_dsa_flush, 1.0f);  in si_blit_dbcb_copy()
151 				si_blitter_end(&sctx->b.b);  in si_blit_dbcb_copy()
163 	sctx->decompression_enabled = false;  in si_blit_dbcb_copy()
164 	sctx->dbcb_depth_copy_enabled = false;  in si_blit_dbcb_copy()
165 	sctx->dbcb_stencil_copy_enabled = false;  in si_blit_dbcb_copy()
166 	si_mark_atom_dirty(sctx, &sctx->db_render_state);  in si_blit_dbcb_copy()
199 si_blit_decompress_zs_planes_in_place(struct si_context *sctx,  in si_blit_decompress_zs_planes_in_place()  argument
212 		sctx->db_flush_stencil_inplace = true;  in si_blit_decompress_zs_planes_in_place()
214 		sctx->db_flush_depth_inplace = true;  in si_blit_decompress_zs_planes_in_place()
215 	si_mark_atom_dirty(sctx, &sctx->db_render_state);  in si_blit_decompress_zs_planes_in_place()
219 	sctx->decompression_enabled = true;  in si_blit_decompress_zs_planes_in_place()
235 			zsurf = sctx->b.b.create_surface(&sctx->b.b, &texture->resource.b.b, &surf_tmpl);  in si_blit_decompress_zs_planes_in_place()
237 			si_blitter_begin(&sctx->b.b, SI_DECOMPRESS);  in si_blit_decompress_zs_planes_in_place()
238 			util_blitter_custom_depth_stencil(sctx->blitter, zsurf, NULL, ~0,  in si_blit_decompress_zs_planes_in_place()
239 							  sctx->custom_dsa_flush,  in si_blit_decompress_zs_planes_in_place()
241 			si_blitter_end(&sctx->b.b);  in si_blit_decompress_zs_planes_in_place()
258 	sctx->decompression_enabled = false;  in si_blit_decompress_zs_planes_in_place()
259 	sctx->db_flush_depth_inplace = false;  in si_blit_decompress_zs_planes_in_place()
260 	sctx->db_flush_stencil_inplace = false;  in si_blit_decompress_zs_planes_in_place()
261 	si_mark_atom_dirty(sctx, &sctx->db_render_state);  in si_blit_decompress_zs_planes_in_place()
268 si_blit_decompress_zs_in_place(struct si_context *sctx,  in si_blit_decompress_zs_in_place()  argument
278 				sctx, texture, PIPE_MASK_Z | PIPE_MASK_S,  in si_blit_decompress_zs_in_place()
288 				sctx, texture, PIPE_MASK_Z,  in si_blit_decompress_zs_in_place()
295 				sctx, texture, PIPE_MASK_S,  in si_blit_decompress_zs_in_place()
302 si_decompress_depth(struct si_context *sctx,  in si_decompress_depth()  argument
335 	if (unlikely(sctx->b.log))  in si_decompress_depth()
336 		u_log_printf(sctx->b.log,  in si_decompress_depth()
346 	     si_init_flushed_depth_texture(&sctx->b.b, &tex->resource.b.b, NULL))) {  in si_decompress_depth()
366 			sctx, tex, dst, copy_planes, levels,  in si_decompress_depth()
384 						sctx, tex,  in si_decompress_depth()
403 		si_make_DB_shader_coherent(sctx, tex->resource.b.b.nr_samples,  in si_decompress_depth()
411 		si_make_CB_shader_coherent(sctx, tex->resource.b.b.nr_samples,  in si_decompress_depth()
416 si_decompress_sampler_depth_textures(struct si_context *sctx,  in si_decompress_sampler_depth_textures()  argument
436 		si_decompress_depth(sctx, tex,  in si_decompress_sampler_depth_textures()
449 	struct si_context *sctx = (struct si_context *)ctx;  in si_blit_decompress_color()  local
460 	if (unlikely(sctx->b.log))  in si_blit_decompress_color()
461 		u_log_printf(sctx->b.log,  in si_blit_decompress_color()
467 		custom_blend = sctx->custom_blend_dcc_decompress;  in si_blit_decompress_color()
477 		custom_blend = sctx->custom_blend_fmask_decompress;  in si_blit_decompress_color()
479 		custom_blend = sctx->custom_blend_eliminate_fastclear;  in si_blit_decompress_color()
482 	sctx->decompression_enabled = true;  in si_blit_decompress_color()
502 			if (custom_blend == sctx->custom_blend_fmask_decompress ||  in si_blit_decompress_color()
503 			    custom_blend == sctx->custom_blend_dcc_decompress)  in si_blit_decompress_color()
504 				sctx->b.flags |= SI_CONTEXT_FLUSH_AND_INV_CB;  in si_blit_decompress_color()
507 			util_blitter_custom_color(sctx->blitter, cbsurf, custom_blend);  in si_blit_decompress_color()
510 			if (custom_blend == sctx->custom_blend_fmask_decompress ||  in si_blit_decompress_color()
511 			    custom_blend == sctx->custom_blend_dcc_decompress)  in si_blit_decompress_color()
512 				sctx->b.flags |= SI_CONTEXT_FLUSH_AND_INV_CB;  in si_blit_decompress_color()
524 	sctx->decompression_enabled = false;  in si_blit_decompress_color()
525 	si_make_CB_shader_coherent(sctx, rtex->resource.b.b.nr_samples,  in si_blit_decompress_color()
530 si_decompress_color_texture(struct si_context *sctx, struct r600_texture *tex,  in si_decompress_color_texture()  argument
537 	si_blit_decompress_color(&sctx->b.b, tex, first_level, last_level, 0,  in si_decompress_color_texture()
543 si_decompress_sampler_color_textures(struct si_context *sctx,  in si_decompress_sampler_color_textures()  argument
560 		si_decompress_color_texture(sctx, tex, view->u.tex.first_level,  in si_decompress_sampler_color_textures()
566 si_decompress_image_color_textures(struct si_context *sctx,  in si_decompress_image_color_textures()  argument
583 		si_decompress_color_texture(sctx, tex, view->u.tex.level,  in si_decompress_image_color_textures()
588 static void si_check_render_feedback_texture(struct si_context *sctx,  in si_check_render_feedback_texture()  argument
600 	for (unsigned j = 0; j < sctx->framebuffer.state.nr_cbufs; ++j) {  in si_check_render_feedback_texture()
603 		if (!sctx->framebuffer.state.cbufs[j])  in si_check_render_feedback_texture()
606 		surf = (struct r600_surface*)sctx->framebuffer.state.cbufs[j];  in si_check_render_feedback_texture()
619 		si_texture_disable_dcc(&sctx->b, tex);  in si_check_render_feedback_texture()
622 static void si_check_render_feedback_textures(struct si_context *sctx,  in si_check_render_feedback_textures()  argument
639 		si_check_render_feedback_texture(sctx, tex,  in si_check_render_feedback_textures()
647 static void si_check_render_feedback_images(struct si_context *sctx,  in si_check_render_feedback_images()  argument
664 		si_check_render_feedback_texture(sctx, tex,  in si_check_render_feedback_images()
672 static void si_check_render_feedback_resident_textures(struct si_context *sctx)  in si_check_render_feedback_resident_textures()  argument
674 	util_dynarray_foreach(&sctx->resident_tex_handles,  in si_check_render_feedback_resident_textures()
685 		si_check_render_feedback_texture(sctx, tex,  in si_check_render_feedback_resident_textures()
693 static void si_check_render_feedback_resident_images(struct si_context *sctx)  in si_check_render_feedback_resident_images()  argument
695 	util_dynarray_foreach(&sctx->resident_img_handles,  in si_check_render_feedback_resident_images()
706 		si_check_render_feedback_texture(sctx, tex,  in si_check_render_feedback_resident_images()
714 static void si_check_render_feedback(struct si_context *sctx)  in si_check_render_feedback()  argument
717 	if (!sctx->need_check_render_feedback)  in si_check_render_feedback()
721 		si_check_render_feedback_images(sctx, &sctx->images[i]);  in si_check_render_feedback()
722 		si_check_render_feedback_textures(sctx, &sctx->samplers[i]);  in si_check_render_feedback()
725 	si_check_render_feedback_resident_images(sctx);  in si_check_render_feedback()
726 	si_check_render_feedback_resident_textures(sctx);  in si_check_render_feedback()
728 	sctx->need_check_render_feedback = false;  in si_check_render_feedback()
731 static void si_decompress_resident_textures(struct si_context *sctx)  in si_decompress_resident_textures()  argument
733 	util_dynarray_foreach(&sctx->resident_tex_needs_color_decompress,  in si_decompress_resident_textures()
738 		si_decompress_color_texture(sctx, tex, view->u.tex.first_level,  in si_decompress_resident_textures()
742 	util_dynarray_foreach(&sctx->resident_tex_needs_depth_decompress,  in si_decompress_resident_textures()
748 		si_decompress_depth(sctx, tex,  in si_decompress_resident_textures()
755 static void si_decompress_resident_images(struct si_context *sctx)  in si_decompress_resident_images()  argument
757 	util_dynarray_foreach(&sctx->resident_img_needs_color_decompress,  in si_decompress_resident_images()
762 		si_decompress_color_texture(sctx, tex, view->u.tex.level,  in si_decompress_resident_images()
767 void si_decompress_textures(struct si_context *sctx, unsigned shader_mask)  in si_decompress_textures()  argument
771 	if (sctx->blitter->running)  in si_decompress_textures()
775 	compressed_colortex_counter = p_atomic_read(&sctx->screen->compressed_colortex_counter);  in si_decompress_textures()
776 	if (compressed_colortex_counter != sctx->b.last_compressed_colortex_counter) {  in si_decompress_textures()
777 		sctx->b.last_compressed_colortex_counter = compressed_colortex_counter;  in si_decompress_textures()
778 		si_update_needs_color_decompress_masks(sctx);  in si_decompress_textures()
782 	mask = sctx->shader_needs_decompress_mask & shader_mask;  in si_decompress_textures()
786 		if (sctx->samplers[i].needs_depth_decompress_mask) {  in si_decompress_textures()
787 			si_decompress_sampler_depth_textures(sctx, &sctx->samplers[i]);  in si_decompress_textures()
789 		if (sctx->samplers[i].needs_color_decompress_mask) {  in si_decompress_textures()
790 			si_decompress_sampler_color_textures(sctx, &sctx->samplers[i]);  in si_decompress_textures()
792 		if (sctx->images[i].needs_color_decompress_mask) {  in si_decompress_textures()
793 			si_decompress_image_color_textures(sctx, &sctx->images[i]);  in si_decompress_textures()
798 		if (sctx->uses_bindless_samplers)  in si_decompress_textures()
799 			si_decompress_resident_textures(sctx);  in si_decompress_textures()
800 		if (sctx->uses_bindless_images)  in si_decompress_textures()
801 			si_decompress_resident_images(sctx);  in si_decompress_textures()
803 		if (sctx->cs_shader_state.program->uses_bindless_samplers)  in si_decompress_textures()
804 			si_decompress_resident_textures(sctx);  in si_decompress_textures()
805 		if (sctx->cs_shader_state.program->uses_bindless_images)  in si_decompress_textures()
806 			si_decompress_resident_images(sctx);  in si_decompress_textures()
809 	si_check_render_feedback(sctx);  in si_decompress_textures()
821 	struct si_context *sctx = (struct si_context *)ctx;  in si_decompress_subresource()  local
834 		if (sctx->framebuffer.state.zsbuf &&  in si_decompress_subresource()
835 		    sctx->framebuffer.state.zsbuf->u.tex.level == level &&  in si_decompress_subresource()
836 		    sctx->framebuffer.state.zsbuf->texture == tex)  in si_decompress_subresource()
837 			si_update_fb_dirtiness_after_rendering(sctx);  in si_decompress_subresource()
839 		si_decompress_depth(sctx, rtex, planes,  in si_decompress_subresource()
847 		for (unsigned i = 0; i < sctx->framebuffer.state.nr_cbufs; i++) {  in si_decompress_subresource()
848 			if (sctx->framebuffer.state.cbufs[i] &&  in si_decompress_subresource()
849 			    sctx->framebuffer.state.cbufs[i]->u.tex.level == level &&  in si_decompress_subresource()
850 			    sctx->framebuffer.state.cbufs[i]->texture == tex) {  in si_decompress_subresource()
851 				si_update_fb_dirtiness_after_rendering(sctx);  in si_decompress_subresource()
879 	struct si_context *sctx = (struct si_context *)ctx;  in si_resource_copy_region()  local
889 		si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width, 0);  in si_resource_copy_region()
908 	util_blitter_default_src_texture(sctx->blitter, &src_templ, src, src_level);  in si_resource_copy_region()
939 	} else if (!util_blitter_is_copy_supported(sctx->blitter, dst, src)) {  in si_resource_copy_region()
1023 	vi_disable_dcc_if_incompatible_format(&sctx->b, dst, dst_level,  in si_resource_copy_region()
1025 	vi_disable_dcc_if_incompatible_format(&sctx->b, src, src_level,  in si_resource_copy_region()
1043 	util_blitter_blit_generic(sctx->blitter, dst_view, &dstbox,  in si_resource_copy_region()
1053 static void si_do_CB_resolve(struct si_context *sctx,  in si_do_CB_resolve()  argument
1060 	sctx->b.flags |= SI_CONTEXT_FLUSH_AND_INV_CB;  in si_do_CB_resolve()
1062 	si_blitter_begin(&sctx->b.b, SI_COLOR_RESOLVE |  in si_do_CB_resolve()
1064 	util_blitter_custom_resolve_color(sctx->blitter, dst, dst_level, dst_z,  in si_do_CB_resolve()
1066 					  ~0, sctx->custom_blend_resolve,  in si_do_CB_resolve()
1068 	si_blitter_end(&sctx->b.b);  in si_do_CB_resolve()
1071 	si_make_CB_shader_coherent(sctx, 1, false);  in si_do_CB_resolve()
1077 	struct si_context *sctx = (struct si_context*)ctx;  in do_hardware_msaa_resolve()  local
1140 			if (sctx->b.chip_class >= GFX9 &&  in do_hardware_msaa_resolve()
1144 			vi_dcc_clear_level(sctx, dst, info->dst.level,  in do_hardware_msaa_resolve()
1150 		si_do_CB_resolve(sctx, info, info->dst.resource,  in do_hardware_msaa_resolve()
1185 	si_do_CB_resolve(sctx, info, tmp, 0, 0, format);  in do_hardware_msaa_resolve()
1194 	util_blitter_blit(sctx->blitter, &blit);  in do_hardware_msaa_resolve()
1204 	struct si_context *sctx = (struct si_context*)ctx;  in si_blit()  local
1218 	    sctx->b.dma_copy &&  in si_blit()
1220 		sctx->b.dma_copy(ctx, info->dst.resource, info->dst.level,  in si_blit()
1228 	assert(util_blitter_is_blit_supported(sctx->blitter, info));  in si_blit()
1232 	vi_disable_dcc_if_incompatible_format(&sctx->b, info->src.resource,  in si_blit()
1235 	vi_disable_dcc_if_incompatible_format(&sctx->b, info->dst.resource,  in si_blit()
1243 	if (sctx->screen->debug_flags & DBG(FORCE_DMA) &&  in si_blit()
1249 	util_blitter_blit(sctx->blitter, info);  in si_blit()
1259 	struct si_context *sctx = (struct si_context*)ctx;  in si_generate_mipmap()  local
1262 	if (!util_blitter_is_copy_supported(sctx->blitter, tex, tex))  in si_generate_mipmap()
1267 	vi_disable_dcc_if_incompatible_format(&sctx->b, tex, base_level,  in si_generate_mipmap()
1277 	sctx->generate_mipmap_for_depth = rtex->is_depth;  in si_generate_mipmap()
1280 	util_blitter_generate_mipmap(sctx->blitter, tex, format,  in si_generate_mipmap()
1285 	sctx->generate_mipmap_for_depth = false;  in si_generate_mipmap()
1326 void si_init_blit_functions(struct si_context *sctx)  in si_init_blit_functions()  argument
1328 	sctx->b.b.resource_copy_region = si_resource_copy_region;  in si_init_blit_functions()
1329 	sctx->b.b.blit = si_blit;  in si_init_blit_functions()
1330 	sctx->b.b.flush_resource = si_flush_resource;  in si_init_blit_functions()
1331 	sctx->b.b.generate_mipmap = si_generate_mipmap;  in si_init_blit_functions()
1332 	sctx->b.blit_decompress_depth = si_blit_decompress_depth;  in si_init_blit_functions()
1333 	sctx->b.decompress_dcc = si_decompress_dcc;  in si_init_blit_functions()