152 	struct si_context *sctx = (struct si_context *)ctx;  in si_create_compute_state()  local
172 		program->compiler_ctx_state.debug = sctx->debug;  in si_create_compute_state()
173 		program->compiler_ctx_state.is_debug_context = sctx->is_debug;  in si_create_compute_state()
179 			(sctx->debug.debug_message && !sctx->debug.async) ||  in si_create_compute_state()
180 			sctx->is_debug ||  in si_create_compute_state()
194 			u_async_debug_drain(&async_debug, &sctx->debug);  in si_create_compute_state()
212 		si_shader_dump(sctx->screen, &program->shader, &sctx->debug,  in si_create_compute_state()
214 		if (si_shader_binary_upload(sctx->screen, &program->shader) < 0) {  in si_create_compute_state()
226 	struct si_context *sctx = (struct si_context*)ctx;  in si_bind_compute_state()  local
229 	sctx->cs_shader_state.program = program;  in si_bind_compute_state()
237 	si_set_active_descriptors(sctx,  in si_bind_compute_state()
241 	si_set_active_descriptors(sctx,  in si_bind_compute_state()
253 	struct si_context *sctx = (struct si_context*)ctx;  in si_set_global_binding()  local
254 	struct si_compute *program = sctx->cs_shader_state.program;  in si_set_global_binding()
277 static void si_initialize_compute(struct si_context *sctx)  in si_initialize_compute()  argument
279 	struct radeon_winsys_cs *cs = sctx->b.gfx.cs;  in si_initialize_compute()
287 	if (sctx->b.chip_class >= CIK) {  in si_initialize_compute()
302 	if (sctx->b.chip_class <= SI) {  in si_initialize_compute()
311 	bc_va = sctx->border_color_buffer->gpu_address;  in si_initialize_compute()
313 	if (sctx->b.chip_class >= CIK) {  in si_initialize_compute()
318 		if (sctx->screen->info.drm_major == 3 ||  in si_initialize_compute()
319 		    (sctx->screen->info.drm_major == 2 &&  in si_initialize_compute()
320 		     sctx->screen->info.drm_minor >= 48)) {  in si_initialize_compute()
326 	sctx->cs_shader_state.emitted_program = NULL;  in si_initialize_compute()
327 	sctx->cs_shader_state.initialized = true;  in si_initialize_compute()
330 static bool si_setup_compute_scratch_buffer(struct si_context *sctx,  in si_setup_compute_scratch_buffer()  argument
336 	scratch_needed = config->scratch_bytes_per_wave * sctx->scratch_waves;  in si_setup_compute_scratch_buffer()
337 	if (sctx->compute_scratch_buffer)  in si_setup_compute_scratch_buffer()
338 		scratch_bo_size = sctx->compute_scratch_buffer->b.b.width0;  in si_setup_compute_scratch_buffer()
341 		r600_resource_reference(&sctx->compute_scratch_buffer, NULL);  in si_setup_compute_scratch_buffer()
343 		sctx->compute_scratch_buffer = (struct r600_resource*)  in si_setup_compute_scratch_buffer()
344 			si_aligned_buffer_create(&sctx->screen->b,  in si_setup_compute_scratch_buffer()
349 		if (!sctx->compute_scratch_buffer)  in si_setup_compute_scratch_buffer()
353 	if (sctx->compute_scratch_buffer != shader->scratch_bo && scratch_needed) {  in si_setup_compute_scratch_buffer()
354 		uint64_t scratch_va = sctx->compute_scratch_buffer->gpu_address;  in si_setup_compute_scratch_buffer()
358 		if (si_shader_binary_upload(sctx->screen, shader))  in si_setup_compute_scratch_buffer()
362 		                        sctx->compute_scratch_buffer);  in si_setup_compute_scratch_buffer()
368 static bool si_switch_compute_shader(struct si_context *sctx,  in si_switch_compute_shader()  argument
374 	struct radeon_winsys_cs *cs = sctx->b.gfx.cs;  in si_switch_compute_shader()
379 	if (sctx->cs_shader_state.emitted_program == program &&  in si_switch_compute_shader()
380 	    sctx->cs_shader_state.offset == offset)  in si_switch_compute_shader()
401 		if (sctx->b.chip_class <= SI) {  in si_switch_compute_shader()
414 	if (!si_setup_compute_scratch_buffer(sctx, shader, config))  in si_switch_compute_shader()
418 		COMPUTE_DBG(sctx->screen, "Waves: %u; Scratch per wave: %u bytes; "  in si_switch_compute_shader()
419 		            "Total Scratch: %u bytes\n", sctx->scratch_waves,  in si_switch_compute_shader()
422 			    sctx->scratch_waves);  in si_switch_compute_shader()
424 		radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,  in si_switch_compute_shader()
436 	if (sctx->b.chip_class >= CIK) {  in si_switch_compute_shader()
437 		cik_prefetch_TC_L2_async(sctx, &program->shader.bo->b.b,  in si_switch_compute_shader()
448 	radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, shader->bo,  in si_switch_compute_shader()
459 	COMPUTE_DBG(sctx->screen, "COMPUTE_PGM_RSRC1: 0x%08x "  in si_switch_compute_shader()
463 	          S_00B860_WAVES(sctx->scratch_waves)  in si_switch_compute_shader()
466 	sctx->cs_shader_state.emitted_program = program;  in si_switch_compute_shader()
467 	sctx->cs_shader_state.offset = offset;  in si_switch_compute_shader()
468 	sctx->cs_shader_state.uses_scratch =  in si_switch_compute_shader()
474 static void setup_scratch_rsrc_user_sgprs(struct si_context *sctx,  in setup_scratch_rsrc_user_sgprs()  argument
478 	struct radeon_winsys_cs *cs = sctx->b.gfx.cs;  in setup_scratch_rsrc_user_sgprs()
479 	uint64_t scratch_va = sctx->compute_scratch_buffer->gpu_address;  in setup_scratch_rsrc_user_sgprs()
496 	if (sctx->b.chip_class >= GFX9) {  in setup_scratch_rsrc_user_sgprs()
501 		if (sctx->b.chip_class < VI) {  in setup_scratch_rsrc_user_sgprs()
517 static void si_setup_user_sgprs_co_v2(struct si_context *sctx,  in si_setup_user_sgprs_co_v2()  argument
522 	struct si_compute *program = sctx->cs_shader_state.program;  in si_setup_user_sgprs_co_v2()
523 	struct radeon_winsys_cs *cs = sctx->b.gfx.cs;  in si_setup_user_sgprs_co_v2()
535 			setup_scratch_rsrc_user_sgprs(sctx, code_object,  in si_setup_user_sgprs_co_v2()
564 		u_upload_data(sctx->b.b.const_uploader, 0, sizeof(dispatch),  in si_setup_user_sgprs_co_v2()
572 		radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, dispatch_buf,  in si_setup_user_sgprs_co_v2()
608 static bool si_upload_compute_input(struct si_context *sctx,  in si_upload_compute_input()  argument
612 	struct radeon_winsys_cs *cs = sctx->b.gfx.cs;  in si_upload_compute_input()
613 	struct si_compute *program = sctx->cs_shader_state.program;  in si_upload_compute_input()
626 	u_upload_alloc(sctx->b.b.const_uploader, 0, kernel_args_size,  in si_upload_compute_input()
627 		       sctx->screen->info.tcc_cache_line_size,  in si_upload_compute_input()
650 		COMPUTE_DBG(sctx->screen, "input %u : %u\n", i,  in si_upload_compute_input()
655 	radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, input_buffer,  in si_upload_compute_input()
659 		si_setup_user_sgprs_co_v2(sctx, code_object, info, kernel_args_va);  in si_upload_compute_input()
672 static void si_setup_tgsi_grid(struct si_context *sctx,  in si_setup_tgsi_grid()  argument
675 	struct si_compute *program = sctx->cs_shader_state.program;  in si_setup_tgsi_grid()
676 	struct radeon_winsys_cs *cs = sctx->b.gfx.cs;  in si_setup_tgsi_grid()
689 			radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,  in si_setup_tgsi_grid()
719 static void si_emit_dispatch_packets(struct si_context *sctx,  in si_emit_dispatch_packets()  argument
722 	struct si_screen *sscreen = sctx->screen;  in si_emit_dispatch_packets()
723 	struct radeon_winsys_cs *cs = sctx->b.gfx.cs;  in si_emit_dispatch_packets()
724 	bool render_cond_bit = sctx->b.render_cond && !sctx->b.render_cond_force_off;  in si_emit_dispatch_packets()
730 	if (sctx->b.chip_class >= CIK) {  in si_emit_dispatch_packets()
755 		S_00B800_ORDER_MODE(sctx->b.chip_class >= CIK);  in si_emit_dispatch_packets()
760 		radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,  in si_emit_dispatch_packets()
788 	struct si_context *sctx = (struct si_context*)ctx;  in si_launch_grid()  local
789 	struct si_compute *program = sctx->cs_shader_state.program;  in si_launch_grid()
800 		(sctx->b.chip_class == SI ||  in si_launch_grid()
801 		 sctx->b.family == CHIP_BONAIRE ||  in si_launch_grid()
802 		 sctx->b.family == CHIP_KABINI) &&  in si_launch_grid()
806 		sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |  in si_launch_grid()
813 	if (sctx->b.last_num_draw_calls != sctx->b.num_draw_calls) {  in si_launch_grid()
814 		si_update_fb_dirtiness_after_rendering(sctx);  in si_launch_grid()
815 		sctx->b.last_num_draw_calls = sctx->b.num_draw_calls;  in si_launch_grid()
818 	si_decompress_textures(sctx, 1 << PIPE_SHADER_COMPUTE);  in si_launch_grid()
828 		if (sctx->b.chip_class <= VI &&  in si_launch_grid()
830 			sctx->b.flags |= SI_CONTEXT_WRITEBACK_GLOBAL_L2;  in si_launch_grid()
835 	si_need_cs_space(sctx);  in si_launch_grid()
837 	if (!sctx->cs_shader_state.initialized)  in si_launch_grid()
838 		si_initialize_compute(sctx);  in si_launch_grid()
840 	if (sctx->b.flags)  in si_launch_grid()
841 		si_emit_cache_flush(sctx);  in si_launch_grid()
843 	if (!si_switch_compute_shader(sctx, program, &program->shader,  in si_launch_grid()
847 	si_upload_compute_shader_descriptors(sctx);  in si_launch_grid()
848 	si_emit_compute_shader_pointers(sctx);  in si_launch_grid()
850 	if (si_is_atom_dirty(sctx, sctx->atoms.s.render_cond)) {  in si_launch_grid()
851 		sctx->atoms.s.render_cond->emit(&sctx->b,  in si_launch_grid()
852 		                                sctx->atoms.s.render_cond);  in si_launch_grid()
853 		si_set_atom_dirty(sctx, sctx->atoms.s.render_cond, false);  in si_launch_grid()
858            unlikely(!si_upload_compute_input(sctx, code_object, info))) {  in si_launch_grid()
869 		radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, buffer,  in si_launch_grid()
875 		si_setup_tgsi_grid(sctx, info);  in si_launch_grid()
877 	si_emit_dispatch_packets(sctx, info);  in si_launch_grid()
879 	if (unlikely(sctx->current_saved_cs)) {  in si_launch_grid()
880 		si_trace_emit(sctx);  in si_launch_grid()
881 		si_log_compute_state(sctx, sctx->b.log);  in si_launch_grid()
884 	sctx->compute_is_busy = true;  in si_launch_grid()
885 	sctx->b.num_compute_calls++;  in si_launch_grid()
886 	if (sctx->cs_shader_state.uses_scratch)  in si_launch_grid()
887 		sctx->b.num_spill_compute_calls++;  in si_launch_grid()
890 		sctx->b.flags |= SI_CONTEXT_CS_PARTIAL_FLUSH;  in si_launch_grid()
907 	struct si_context *sctx = (struct si_context*)ctx;  in si_delete_compute_state()  local
912 	if (program == sctx->cs_shader_state.program)  in si_delete_compute_state()
913 		sctx->cs_shader_state.program = NULL;  in si_delete_compute_state()
915 	if (program == sctx->cs_shader_state.emitted_program)  in si_delete_compute_state()
916 		sctx->cs_shader_state.emitted_program = NULL;  in si_delete_compute_state()
925 void si_init_compute_functions(struct si_context *sctx)  in si_init_compute_functions()  argument
927 	sctx->b.b.create_compute_state = si_create_compute_state;  in si_init_compute_functions()
928 	sctx->b.b.delete_compute_state = si_delete_compute_state;  in si_init_compute_functions()
929 	sctx->b.b.bind_compute_state = si_bind_compute_state;  in si_init_compute_functions()
931 	sctx->b.b.set_compute_resources = si_set_compute_resources;  in si_init_compute_functions()
932 	sctx->b.b.set_global_binding = si_set_global_binding;  in si_init_compute_functions()
933 	sctx->b.b.launch_grid = si_launch_grid;  in si_init_compute_functions()