1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based on si_state.c
6 * Copyright © 2015 Advanced Micro Devices, Inc.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
17 * Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * IN THE SOFTWARE.
26 */
27
28 /* command buffer handling for AMD GCN */
29
30 #include "radv_private.h"
31 #include "radv_shader.h"
32 #include "radv_cs.h"
33 #include "sid.h"
34 #include "radv_util.h"
35
36 static void
si_write_harvested_raster_configs(struct radv_physical_device * physical_device,struct radeon_cmdbuf * cs,unsigned raster_config,unsigned raster_config_1)37 si_write_harvested_raster_configs(struct radv_physical_device *physical_device,
38 struct radeon_cmdbuf *cs,
39 unsigned raster_config,
40 unsigned raster_config_1)
41 {
42 unsigned num_se = MAX2(physical_device->rad_info.max_se, 1);
43 unsigned raster_config_se[4];
44 unsigned se;
45
46 ac_get_harvested_configs(&physical_device->rad_info,
47 raster_config,
48 &raster_config_1,
49 raster_config_se);
50
51 for (se = 0; se < num_se; se++) {
52 /* GRBM_GFX_INDEX has a different offset on GFX6 and GFX7+ */
53 if (physical_device->rad_info.chip_class < GFX7)
54 radeon_set_config_reg(cs, R_00802C_GRBM_GFX_INDEX,
55 S_00802C_SE_INDEX(se) |
56 S_00802C_SH_BROADCAST_WRITES(1) |
57 S_00802C_INSTANCE_BROADCAST_WRITES(1));
58 else
59 radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX,
60 S_030800_SE_INDEX(se) | S_030800_SH_BROADCAST_WRITES(1) |
61 S_030800_INSTANCE_BROADCAST_WRITES(1));
62 radeon_set_context_reg(cs, R_028350_PA_SC_RASTER_CONFIG, raster_config_se[se]);
63 }
64
65 /* GRBM_GFX_INDEX has a different offset on GFX6 and GFX7+ */
66 if (physical_device->rad_info.chip_class < GFX7)
67 radeon_set_config_reg(cs, R_00802C_GRBM_GFX_INDEX,
68 S_00802C_SE_BROADCAST_WRITES(1) |
69 S_00802C_SH_BROADCAST_WRITES(1) |
70 S_00802C_INSTANCE_BROADCAST_WRITES(1));
71 else
72 radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX,
73 S_030800_SE_BROADCAST_WRITES(1) | S_030800_SH_BROADCAST_WRITES(1) |
74 S_030800_INSTANCE_BROADCAST_WRITES(1));
75
76 if (physical_device->rad_info.chip_class >= GFX7)
77 radeon_set_context_reg(cs, R_028354_PA_SC_RASTER_CONFIG_1, raster_config_1);
78 }
79
80 void
si_emit_compute(struct radv_device * device,struct radeon_cmdbuf * cs)81 si_emit_compute(struct radv_device *device,
82 struct radeon_cmdbuf *cs)
83 {
84 radeon_set_sh_reg_seq(cs, R_00B810_COMPUTE_START_X, 3);
85 radeon_emit(cs, 0);
86 radeon_emit(cs, 0);
87 radeon_emit(cs, 0);
88
89 radeon_set_sh_reg_seq(cs, R_00B858_COMPUTE_STATIC_THREAD_MGMT_SE0, 2);
90 /* R_00B858_COMPUTE_STATIC_THREAD_MGMT_SE0 / SE1,
91 * renamed COMPUTE_DESTINATION_EN_SEn on gfx10. */
92 radeon_emit(cs, S_00B858_SH0_CU_EN(0xffff) | S_00B858_SH1_CU_EN(0xffff));
93 radeon_emit(cs, S_00B858_SH0_CU_EN(0xffff) | S_00B858_SH1_CU_EN(0xffff));
94
95 if (device->physical_device->rad_info.chip_class >= GFX7) {
96 /* Also set R_00B858_COMPUTE_STATIC_THREAD_MGMT_SE2 / SE3 */
97 radeon_set_sh_reg_seq(cs,
98 R_00B864_COMPUTE_STATIC_THREAD_MGMT_SE2, 2);
99 radeon_emit(cs, S_00B858_SH0_CU_EN(0xffff) |
100 S_00B858_SH1_CU_EN(0xffff));
101 radeon_emit(cs, S_00B858_SH0_CU_EN(0xffff) |
102 S_00B858_SH1_CU_EN(0xffff));
103
104 if (device->border_color_data.bo) {
105 uint64_t bc_va = radv_buffer_get_va(device->border_color_data.bo);
106
107 radeon_set_uconfig_reg_seq(cs, R_030E00_TA_CS_BC_BASE_ADDR, 2);
108 radeon_emit(cs, bc_va >> 8);
109 radeon_emit(cs, S_030E04_ADDRESS(bc_va >> 40));
110 }
111 }
112
113 if (device->physical_device->rad_info.chip_class >= GFX9) {
114 radeon_set_uconfig_reg(cs, R_0301EC_CP_COHER_START_DELAY,
115 device->physical_device->rad_info.chip_class >= GFX10 ? 0x20 : 0);
116 }
117
118 if (device->physical_device->rad_info.chip_class >= GFX10) {
119 radeon_set_sh_reg(cs, R_00B890_COMPUTE_USER_ACCUM_0, 0);
120 radeon_set_sh_reg(cs, R_00B894_COMPUTE_USER_ACCUM_1, 0);
121 radeon_set_sh_reg(cs, R_00B898_COMPUTE_USER_ACCUM_2, 0);
122 radeon_set_sh_reg(cs, R_00B89C_COMPUTE_USER_ACCUM_3, 0);
123 radeon_set_sh_reg(cs, R_00B8A0_COMPUTE_PGM_RSRC3, 0);
124 radeon_set_sh_reg(cs, R_00B9F4_COMPUTE_DISPATCH_TUNNEL, 0);
125 }
126
127 /* This register has been moved to R_00CD20_COMPUTE_MAX_WAVE_ID
128 * and is now per pipe, so it should be handled in the
129 * kernel if we want to use something other than the default value,
130 * which is now 0x22f.
131 */
132 if (device->physical_device->rad_info.chip_class <= GFX6) {
133 /* XXX: This should be:
134 * (number of compute units) * 4 * (waves per simd) - 1 */
135
136 radeon_set_sh_reg(cs, R_00B82C_COMPUTE_MAX_WAVE_ID,
137 0x190 /* Default value */);
138
139 if (device->border_color_data.bo) {
140 uint64_t bc_va = radv_buffer_get_va(device->border_color_data.bo);
141 radeon_set_config_reg(cs, R_00950C_TA_CS_BC_BASE_ADDR, bc_va >> 8);
142 }
143 }
144 }
145
146 /* 12.4 fixed-point */
radv_pack_float_12p4(float x)147 static unsigned radv_pack_float_12p4(float x)
148 {
149 return x <= 0 ? 0 :
150 x >= 4096 ? 0xffff : x * 16;
151 }
152
153 static void
si_set_raster_config(struct radv_physical_device * physical_device,struct radeon_cmdbuf * cs)154 si_set_raster_config(struct radv_physical_device *physical_device,
155 struct radeon_cmdbuf *cs)
156 {
157 unsigned num_rb = MIN2(physical_device->rad_info.num_render_backends, 16);
158 unsigned rb_mask = physical_device->rad_info.enabled_rb_mask;
159 unsigned raster_config, raster_config_1;
160
161 ac_get_raster_config(&physical_device->rad_info,
162 &raster_config,
163 &raster_config_1, NULL);
164
165 /* Always use the default config when all backends are enabled
166 * (or when we failed to determine the enabled backends).
167 */
168 if (!rb_mask || util_bitcount(rb_mask) >= num_rb) {
169 radeon_set_context_reg(cs, R_028350_PA_SC_RASTER_CONFIG,
170 raster_config);
171 if (physical_device->rad_info.chip_class >= GFX7)
172 radeon_set_context_reg(cs, R_028354_PA_SC_RASTER_CONFIG_1,
173 raster_config_1);
174 } else {
175 si_write_harvested_raster_configs(physical_device, cs,
176 raster_config,
177 raster_config_1);
178 }
179 }
180
181 void
si_emit_graphics(struct radv_device * device,struct radeon_cmdbuf * cs)182 si_emit_graphics(struct radv_device *device,
183 struct radeon_cmdbuf *cs)
184 {
185 struct radv_physical_device *physical_device = device->physical_device;
186
187 bool has_clear_state = physical_device->rad_info.has_clear_state;
188 int i;
189
190 radeon_emit(cs, PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
191 radeon_emit(cs, CC0_UPDATE_LOAD_ENABLES(1));
192 radeon_emit(cs, CC1_UPDATE_SHADOW_ENABLES(1));
193
194 if (has_clear_state) {
195 radeon_emit(cs, PKT3(PKT3_CLEAR_STATE, 0, 0));
196 radeon_emit(cs, 0);
197 }
198
199 if (physical_device->rad_info.chip_class <= GFX8)
200 si_set_raster_config(physical_device, cs);
201
202 radeon_set_context_reg(cs, R_028A18_VGT_HOS_MAX_TESS_LEVEL, fui(64));
203 if (!has_clear_state)
204 radeon_set_context_reg(cs, R_028A1C_VGT_HOS_MIN_TESS_LEVEL, fui(0));
205
206 /* FIXME calculate these values somehow ??? */
207 if (physical_device->rad_info.chip_class <= GFX8) {
208 radeon_set_context_reg(cs, R_028A54_VGT_GS_PER_ES, SI_GS_PER_ES);
209 radeon_set_context_reg(cs, R_028A58_VGT_ES_PER_GS, 0x40);
210 }
211
212 if (!has_clear_state) {
213 radeon_set_context_reg(cs, R_028A5C_VGT_GS_PER_VS, 0x2);
214 radeon_set_context_reg(cs, R_028A8C_VGT_PRIMITIVEID_RESET, 0x0);
215 radeon_set_context_reg(cs, R_028B98_VGT_STRMOUT_BUFFER_CONFIG, 0x0);
216 }
217
218 if (physical_device->rad_info.chip_class <= GFX9)
219 radeon_set_context_reg(cs, R_028AA0_VGT_INSTANCE_STEP_RATE_0, 1);
220 if (!has_clear_state)
221 radeon_set_context_reg(cs, R_028AB8_VGT_VTX_CNT_EN, 0x0);
222 if (physical_device->rad_info.chip_class < GFX7)
223 radeon_set_config_reg(cs, R_008A14_PA_CL_ENHANCE, S_008A14_NUM_CLIP_SEQ(3) |
224 S_008A14_CLIP_VTX_REORDER_ENA(1));
225
226 if (!has_clear_state)
227 radeon_set_context_reg(cs, R_02882C_PA_SU_PRIM_FILTER_CNTL, 0);
228
229 /* CLEAR_STATE doesn't clear these correctly on certain generations.
230 * I don't know why. Deduced by trial and error.
231 */
232 if (physical_device->rad_info.chip_class <= GFX7 || !has_clear_state) {
233 radeon_set_context_reg(cs, R_028B28_VGT_STRMOUT_DRAW_OPAQUE_OFFSET, 0);
234 radeon_set_context_reg(cs, R_028204_PA_SC_WINDOW_SCISSOR_TL,
235 S_028204_WINDOW_OFFSET_DISABLE(1));
236 radeon_set_context_reg(cs, R_028240_PA_SC_GENERIC_SCISSOR_TL,
237 S_028240_WINDOW_OFFSET_DISABLE(1));
238 radeon_set_context_reg(cs, R_028244_PA_SC_GENERIC_SCISSOR_BR,
239 S_028244_BR_X(16384) | S_028244_BR_Y(16384));
240 radeon_set_context_reg(cs, R_028030_PA_SC_SCREEN_SCISSOR_TL, 0);
241 radeon_set_context_reg(cs, R_028034_PA_SC_SCREEN_SCISSOR_BR,
242 S_028034_BR_X(16384) | S_028034_BR_Y(16384));
243 }
244
245 if (!has_clear_state) {
246 for (i = 0; i < 16; i++) {
247 radeon_set_context_reg(cs, R_0282D0_PA_SC_VPORT_ZMIN_0 + i*8, 0);
248 radeon_set_context_reg(cs, R_0282D4_PA_SC_VPORT_ZMAX_0 + i*8, fui(1.0));
249 }
250 }
251
252 if (!has_clear_state) {
253 radeon_set_context_reg(cs, R_02820C_PA_SC_CLIPRECT_RULE, 0xFFFF);
254 radeon_set_context_reg(cs, R_028230_PA_SC_EDGERULE, 0xAAAAAAAA);
255 /* PA_SU_HARDWARE_SCREEN_OFFSET must be 0 due to hw bug on GFX6 */
256 radeon_set_context_reg(cs, R_028234_PA_SU_HARDWARE_SCREEN_OFFSET, 0);
257 radeon_set_context_reg(cs, R_028820_PA_CL_NANINF_CNTL, 0);
258 radeon_set_context_reg(cs, R_028AC0_DB_SRESULTS_COMPARE_STATE0, 0x0);
259 radeon_set_context_reg(cs, R_028AC4_DB_SRESULTS_COMPARE_STATE1, 0x0);
260 radeon_set_context_reg(cs, R_028AC8_DB_PRELOAD_CONTROL, 0x0);
261 }
262
263 radeon_set_context_reg(cs, R_02800C_DB_RENDER_OVERRIDE,
264 S_02800C_FORCE_HIS_ENABLE0(V_02800C_FORCE_DISABLE) |
265 S_02800C_FORCE_HIS_ENABLE1(V_02800C_FORCE_DISABLE));
266
267 if (physical_device->rad_info.chip_class >= GFX10) {
268 radeon_set_context_reg(cs, R_028A98_VGT_DRAW_PAYLOAD_CNTL, 0);
269 radeon_set_uconfig_reg(cs, R_030964_GE_MAX_VTX_INDX, ~0);
270 radeon_set_uconfig_reg(cs, R_030924_GE_MIN_VTX_INDX, 0);
271 radeon_set_uconfig_reg(cs, R_030928_GE_INDX_OFFSET, 0);
272 radeon_set_uconfig_reg(cs, R_03097C_GE_STEREO_CNTL, 0);
273 radeon_set_uconfig_reg(cs, R_030988_GE_USER_VGPR_EN, 0);
274 } else if (physical_device->rad_info.chip_class == GFX9) {
275 radeon_set_uconfig_reg(cs, R_030920_VGT_MAX_VTX_INDX, ~0);
276 radeon_set_uconfig_reg(cs, R_030924_VGT_MIN_VTX_INDX, 0);
277 radeon_set_uconfig_reg(cs, R_030928_VGT_INDX_OFFSET, 0);
278 } else {
279 /* These registers, when written, also overwrite the
280 * CLEAR_STATE context, so we can't rely on CLEAR_STATE setting
281 * them. It would be an issue if there was another UMD
282 * changing them.
283 */
284 radeon_set_context_reg(cs, R_028400_VGT_MAX_VTX_INDX, ~0);
285 radeon_set_context_reg(cs, R_028404_VGT_MIN_VTX_INDX, 0);
286 radeon_set_context_reg(cs, R_028408_VGT_INDX_OFFSET, 0);
287 }
288
289 unsigned cu_mask_ps = 0xffffffff;
290
291 /* It's wasteful to enable all CUs for PS if shader arrays have a
292 * different number of CUs. The reason is that the hardware sends the
293 * same number of PS waves to each shader array, so the slowest shader
294 * array limits the performance. Disable the extra CUs for PS in
295 * other shader arrays to save power and thus increase clocks for busy
296 * CUs. In the future, we might disable or enable this tweak only for
297 * certain apps.
298 */
299 if (physical_device->rad_info.chip_class >= GFX10_3)
300 cu_mask_ps = u_bit_consecutive(0, physical_device->rad_info.min_good_cu_per_sa);
301
302 if (physical_device->rad_info.chip_class >= GFX7) {
303 if (physical_device->rad_info.chip_class >= GFX10) {
304 /* Logical CUs 16 - 31 */
305 radeon_set_sh_reg_idx(physical_device, cs, R_00B404_SPI_SHADER_PGM_RSRC4_HS,
306 3, S_00B404_CU_EN(0xffff));
307 radeon_set_sh_reg_idx(physical_device, cs, R_00B104_SPI_SHADER_PGM_RSRC4_VS,
308 3, S_00B104_CU_EN(0xffff));
309 radeon_set_sh_reg_idx(physical_device, cs, R_00B004_SPI_SHADER_PGM_RSRC4_PS,
310 3, S_00B004_CU_EN(cu_mask_ps >> 16));
311 }
312
313 if (physical_device->rad_info.chip_class >= GFX9) {
314 radeon_set_sh_reg_idx(physical_device, cs, R_00B41C_SPI_SHADER_PGM_RSRC3_HS,
315 3, S_00B41C_CU_EN(0xffff) | S_00B41C_WAVE_LIMIT(0x3F));
316 } else {
317 radeon_set_sh_reg(cs, R_00B51C_SPI_SHADER_PGM_RSRC3_LS,
318 S_00B51C_CU_EN(0xffff) | S_00B51C_WAVE_LIMIT(0x3F));
319 radeon_set_sh_reg(cs, R_00B41C_SPI_SHADER_PGM_RSRC3_HS,
320 S_00B41C_WAVE_LIMIT(0x3F));
321 radeon_set_sh_reg(cs, R_00B31C_SPI_SHADER_PGM_RSRC3_ES,
322 S_00B31C_CU_EN(0xffff) | S_00B31C_WAVE_LIMIT(0x3F));
323 /* If this is 0, Bonaire can hang even if GS isn't being used.
324 * Other chips are unaffected. These are suboptimal values,
325 * but we don't use on-chip GS.
326 */
327 radeon_set_context_reg(cs, R_028A44_VGT_GS_ONCHIP_CNTL,
328 S_028A44_ES_VERTS_PER_SUBGRP(64) |
329 S_028A44_GS_PRIMS_PER_SUBGRP(4));
330 }
331
332 /* Compute LATE_ALLOC_VS.LIMIT. */
333 unsigned num_cu_per_sh = physical_device->rad_info.min_good_cu_per_sa;
334 unsigned late_alloc_wave64 = 0; /* The limit is per SA. */
335 unsigned late_alloc_wave64_gs = 0;
336 unsigned cu_mask_vs = 0xffff;
337 unsigned cu_mask_gs = 0xffff;
338
339 if (physical_device->rad_info.chip_class >= GFX10) {
340 /* For Wave32, the hw will launch twice the number of late
341 * alloc waves, so 1 == 2x wave32.
342 */
343 if (!physical_device->rad_info.use_late_alloc) {
344 late_alloc_wave64 = 0;
345 } else if (num_cu_per_sh <= 6) {
346 late_alloc_wave64 = num_cu_per_sh - 2;
347 } else {
348 late_alloc_wave64 = (num_cu_per_sh - 2) * 4;
349
350 /* CU2 & CU3 disabled because of the dual CU design */
351 cu_mask_vs = 0xfff3;
352 cu_mask_gs = 0xfff3; /* NGG only */
353 }
354
355 late_alloc_wave64_gs = late_alloc_wave64;
356
357 /* Don't use late alloc for NGG on Navi14 due to a hw
358 * bug. If NGG is never used, enable all CUs.
359 */
360 if (!physical_device->use_ngg ||
361 physical_device->rad_info.family == CHIP_NAVI14) {
362 late_alloc_wave64_gs = 0;
363 cu_mask_gs = 0xffff;
364 }
365
366 /* Limit LATE_ALLOC_GS for prevent a hang (hw bug). */
367 if (physical_device->rad_info.chip_class == GFX10)
368 late_alloc_wave64_gs = MIN2(late_alloc_wave64_gs, 64);
369 } else {
370 if (!physical_device->rad_info.use_late_alloc) {
371 late_alloc_wave64 = 0;
372 } else if (num_cu_per_sh <= 4) {
373 /* Too few available compute units per SA.
374 * Disallowing VS to run on one CU could hurt
375 * us more than late VS allocation would help.
376 *
377 * 2 is the highest safe number that allows us
378 * to keep all CUs enabled.
379 */
380 late_alloc_wave64 = 2;
381 } else {
382 /* This is a good initial value, allowing 1
383 * late_alloc wave per SIMD on num_cu - 2.
384 */
385 late_alloc_wave64 = (num_cu_per_sh - 2) * 4;
386 }
387
388 if (late_alloc_wave64 > 2)
389 cu_mask_vs = 0xfffe; /* 1 CU disabled */
390 }
391
392 radeon_set_sh_reg_idx(physical_device, cs, R_00B118_SPI_SHADER_PGM_RSRC3_VS,
393 3, S_00B118_CU_EN(cu_mask_vs) |
394 S_00B118_WAVE_LIMIT(0x3F));
395 radeon_set_sh_reg(cs, R_00B11C_SPI_SHADER_LATE_ALLOC_VS,
396 S_00B11C_LIMIT(late_alloc_wave64));
397
398 radeon_set_sh_reg_idx(physical_device, cs, R_00B21C_SPI_SHADER_PGM_RSRC3_GS,
399 3, S_00B21C_CU_EN(cu_mask_gs) | S_00B21C_WAVE_LIMIT(0x3F));
400
401 if (physical_device->rad_info.chip_class >= GFX10) {
402 radeon_set_sh_reg_idx(physical_device, cs, R_00B204_SPI_SHADER_PGM_RSRC4_GS,
403 3, S_00B204_CU_EN(0xffff) |
404 S_00B204_SPI_SHADER_LATE_ALLOC_GS_GFX10(late_alloc_wave64_gs));
405 }
406
407 radeon_set_sh_reg_idx(physical_device, cs, R_00B01C_SPI_SHADER_PGM_RSRC3_PS,
408 3, S_00B01C_CU_EN(cu_mask_ps) | S_00B01C_WAVE_LIMIT(0x3F));
409 }
410
411 if (physical_device->rad_info.chip_class >= GFX10) {
412 /* Break up a pixel wave if it contains deallocs for more than
413 * half the parameter cache.
414 *
415 * To avoid a deadlock where pixel waves aren't launched
416 * because they're waiting for more pixels while the frontend
417 * is stuck waiting for PC space, the maximum allowed value is
418 * the size of the PC minus the largest possible allocation for
419 * a single primitive shader subgroup.
420 */
421 radeon_set_context_reg(cs, R_028C50_PA_SC_NGG_MODE_CNTL,
422 S_028C50_MAX_DEALLOCS_IN_WAVE(512));
423 radeon_set_context_reg(cs, R_028C58_VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
424
425 /* Enable CMASK/FMASK/HTILE/DCC caching in L2 for small chips. */
426 unsigned meta_write_policy, meta_read_policy;
427
428 /* TODO: investigate whether LRU improves performance on other chips too */
429 if (physical_device->rad_info.num_render_backends <= 4) {
430 meta_write_policy = V_02807C_CACHE_LRU_WR; /* cache writes */
431 meta_read_policy = V_02807C_CACHE_LRU_RD; /* cache reads */
432 } else {
433 meta_write_policy = V_02807C_CACHE_STREAM; /* write combine */
434 meta_read_policy = V_02807C_CACHE_NOA; /* don't cache reads */
435 }
436
437 radeon_set_context_reg(cs, R_02807C_DB_RMI_L2_CACHE_CONTROL,
438 S_02807C_Z_WR_POLICY(V_02807C_CACHE_STREAM) |
439 S_02807C_S_WR_POLICY(V_02807C_CACHE_STREAM) |
440 S_02807C_HTILE_WR_POLICY(meta_write_policy) |
441 S_02807C_ZPCPSD_WR_POLICY(V_02807C_CACHE_STREAM) |
442 S_02807C_Z_RD_POLICY(V_02807C_CACHE_NOA) |
443 S_02807C_S_RD_POLICY(V_02807C_CACHE_NOA) |
444 S_02807C_HTILE_RD_POLICY(meta_read_policy));
445
446 radeon_set_context_reg(cs, R_028410_CB_RMI_GL2_CACHE_CONTROL,
447 S_028410_CMASK_WR_POLICY(meta_write_policy) |
448 S_028410_FMASK_WR_POLICY(meta_write_policy) |
449 S_028410_DCC_WR_POLICY(meta_write_policy) |
450 S_028410_COLOR_WR_POLICY(V_028410_CACHE_STREAM) |
451 S_028410_CMASK_RD_POLICY(meta_read_policy) |
452 S_028410_FMASK_RD_POLICY(meta_read_policy) |
453 S_028410_DCC_RD_POLICY(meta_read_policy) |
454 S_028410_COLOR_RD_POLICY(V_028410_CACHE_NOA));
455 radeon_set_context_reg(cs, R_028428_CB_COVERAGE_OUT_CONTROL, 0);
456
457 radeon_set_sh_reg(cs, R_00B0C8_SPI_SHADER_USER_ACCUM_PS_0, 0);
458 radeon_set_sh_reg(cs, R_00B0CC_SPI_SHADER_USER_ACCUM_PS_1, 0);
459 radeon_set_sh_reg(cs, R_00B0D0_SPI_SHADER_USER_ACCUM_PS_2, 0);
460 radeon_set_sh_reg(cs, R_00B0D4_SPI_SHADER_USER_ACCUM_PS_3, 0);
461 radeon_set_sh_reg(cs, R_00B1C8_SPI_SHADER_USER_ACCUM_VS_0, 0);
462 radeon_set_sh_reg(cs, R_00B1CC_SPI_SHADER_USER_ACCUM_VS_1, 0);
463 radeon_set_sh_reg(cs, R_00B1D0_SPI_SHADER_USER_ACCUM_VS_2, 0);
464 radeon_set_sh_reg(cs, R_00B1D4_SPI_SHADER_USER_ACCUM_VS_3, 0);
465 radeon_set_sh_reg(cs, R_00B2C8_SPI_SHADER_USER_ACCUM_ESGS_0, 0);
466 radeon_set_sh_reg(cs, R_00B2CC_SPI_SHADER_USER_ACCUM_ESGS_1, 0);
467 radeon_set_sh_reg(cs, R_00B2D0_SPI_SHADER_USER_ACCUM_ESGS_2, 0);
468 radeon_set_sh_reg(cs, R_00B2D4_SPI_SHADER_USER_ACCUM_ESGS_3, 0);
469 radeon_set_sh_reg(cs, R_00B4C8_SPI_SHADER_USER_ACCUM_LSHS_0, 0);
470 radeon_set_sh_reg(cs, R_00B4CC_SPI_SHADER_USER_ACCUM_LSHS_1, 0);
471 radeon_set_sh_reg(cs, R_00B4D0_SPI_SHADER_USER_ACCUM_LSHS_2, 0);
472 radeon_set_sh_reg(cs, R_00B4D4_SPI_SHADER_USER_ACCUM_LSHS_3, 0);
473
474 radeon_set_sh_reg(cs, R_00B0C0_SPI_SHADER_REQ_CTRL_PS,
475 S_00B0C0_SOFT_GROUPING_EN(1) |
476 S_00B0C0_NUMBER_OF_REQUESTS_PER_CU(4 - 1));
477 radeon_set_sh_reg(cs, R_00B1C0_SPI_SHADER_REQ_CTRL_VS, 0);
478
479 if (physical_device->rad_info.chip_class >= GFX10_3) {
480 radeon_set_context_reg(cs, R_028750_SX_PS_DOWNCONVERT_CONTROL, 0xff);
481 /* This allows sample shading. */
482 radeon_set_context_reg(cs, R_028848_PA_CL_VRS_CNTL,
483 S_028848_SAMPLE_ITER_COMBINER_MODE(1));
484 }
485
486 if (physical_device->rad_info.chip_class == GFX10) {
487 /* SQ_NON_EVENT must be emitted before GE_PC_ALLOC is written. */
488 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
489 radeon_emit(cs, EVENT_TYPE(V_028A90_SQ_NON_EVENT) | EVENT_INDEX(0));
490 }
491
492 /* TODO: For culling, replace 128 with 256. */
493 radeon_set_uconfig_reg(cs, R_030980_GE_PC_ALLOC,
494 S_030980_OVERSUB_EN(physical_device->rad_info.use_late_alloc) |
495 S_030980_NUM_PC_LINES(128 * physical_device->rad_info.max_se - 1));
496 }
497
498 if (physical_device->rad_info.chip_class >= GFX9) {
499 radeon_set_context_reg(cs, R_028B50_VGT_TESS_DISTRIBUTION,
500 S_028B50_ACCUM_ISOLINE(40) |
501 S_028B50_ACCUM_TRI(30) |
502 S_028B50_ACCUM_QUAD(24) |
503 S_028B50_DONUT_SPLIT(24) |
504 S_028B50_TRAP_SPLIT(6));
505 } else if (physical_device->rad_info.chip_class >= GFX8) {
506 uint32_t vgt_tess_distribution;
507
508 vgt_tess_distribution = S_028B50_ACCUM_ISOLINE(32) |
509 S_028B50_ACCUM_TRI(11) |
510 S_028B50_ACCUM_QUAD(11) |
511 S_028B50_DONUT_SPLIT(16);
512
513 if (physical_device->rad_info.family == CHIP_FIJI ||
514 physical_device->rad_info.family >= CHIP_POLARIS10)
515 vgt_tess_distribution |= S_028B50_TRAP_SPLIT(3);
516
517 radeon_set_context_reg(cs, R_028B50_VGT_TESS_DISTRIBUTION,
518 vgt_tess_distribution);
519 } else if (!has_clear_state) {
520 radeon_set_context_reg(cs, R_028C58_VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
521 radeon_set_context_reg(cs, R_028C5C_VGT_OUT_DEALLOC_CNTL, 16);
522 }
523
524 if (device->border_color_data.bo) {
525 uint64_t border_color_va = radv_buffer_get_va(device->border_color_data.bo);
526
527 radeon_set_context_reg(cs, R_028080_TA_BC_BASE_ADDR, border_color_va >> 8);
528 if (physical_device->rad_info.chip_class >= GFX7) {
529 radeon_set_context_reg(cs, R_028084_TA_BC_BASE_ADDR_HI,
530 S_028084_ADDRESS(border_color_va >> 40));
531 }
532 }
533
534 if (physical_device->rad_info.chip_class >= GFX9) {
535 radeon_set_context_reg(cs, R_028C48_PA_SC_BINNER_CNTL_1,
536 S_028C48_MAX_ALLOC_COUNT(physical_device->rad_info.pbb_max_alloc_count - 1) |
537 S_028C48_MAX_PRIM_PER_BATCH(1023));
538 radeon_set_context_reg(cs, R_028C4C_PA_SC_CONSERVATIVE_RASTERIZATION_CNTL,
539 S_028C4C_NULL_SQUAD_AA_MASK_ENABLE(1));
540 radeon_set_uconfig_reg(cs, R_030968_VGT_INSTANCE_BASE_ID, 0);
541 }
542
543 unsigned tmp = (unsigned)(1.0 * 8.0);
544 radeon_set_context_reg_seq(cs, R_028A00_PA_SU_POINT_SIZE, 1);
545 radeon_emit(cs, S_028A00_HEIGHT(tmp) | S_028A00_WIDTH(tmp));
546 radeon_set_context_reg_seq(cs, R_028A04_PA_SU_POINT_MINMAX, 1);
547 radeon_emit(cs, S_028A04_MIN_SIZE(radv_pack_float_12p4(0)) |
548 S_028A04_MAX_SIZE(radv_pack_float_12p4(8191.875/2)));
549
550 if (!has_clear_state) {
551 radeon_set_context_reg(cs, R_028004_DB_COUNT_CONTROL,
552 S_028004_ZPASS_INCREMENT_DISABLE(1));
553 }
554
555 /* Enable the Polaris small primitive filter control.
556 * XXX: There is possibly an issue when MSAA is off (see RadeonSI
557 * has_msaa_sample_loc_bug). But this doesn't seem to regress anything,
558 * and AMDVLK doesn't have a workaround as well.
559 */
560 if (physical_device->rad_info.family >= CHIP_POLARIS10) {
561 unsigned small_prim_filter_cntl =
562 S_028830_SMALL_PRIM_FILTER_ENABLE(1) |
563 /* Workaround for a hw line bug. */
564 S_028830_LINE_FILTER_DISABLE(physical_device->rad_info.family <= CHIP_POLARIS12);
565
566 radeon_set_context_reg(cs, R_028830_PA_SU_SMALL_PRIM_FILTER_CNTL,
567 small_prim_filter_cntl);
568 }
569
570 radeon_set_context_reg(cs, R_0286D4_SPI_INTERP_CONTROL_0,
571 S_0286D4_FLAT_SHADE_ENA(1) |
572 S_0286D4_PNT_SPRITE_ENA(1) |
573 S_0286D4_PNT_SPRITE_OVRD_X(V_0286D4_SPI_PNT_SPRITE_SEL_S) |
574 S_0286D4_PNT_SPRITE_OVRD_Y(V_0286D4_SPI_PNT_SPRITE_SEL_T) |
575 S_0286D4_PNT_SPRITE_OVRD_Z(V_0286D4_SPI_PNT_SPRITE_SEL_0) |
576 S_0286D4_PNT_SPRITE_OVRD_W(V_0286D4_SPI_PNT_SPRITE_SEL_1) |
577 S_0286D4_PNT_SPRITE_TOP_1(0)); /* vulkan is top to bottom - 1.0 at bottom */
578
579 radeon_set_context_reg(cs, R_028BE4_PA_SU_VTX_CNTL,
580 S_028BE4_PIX_CENTER(1) |
581 S_028BE4_ROUND_MODE(V_028BE4_X_ROUND_TO_EVEN) |
582 S_028BE4_QUANT_MODE(V_028BE4_X_16_8_FIXED_POINT_1_256TH));
583
584 radeon_set_context_reg(cs, R_028818_PA_CL_VTE_CNTL,
585 S_028818_VTX_W0_FMT(1) |
586 S_028818_VPORT_X_SCALE_ENA(1) | S_028818_VPORT_X_OFFSET_ENA(1) |
587 S_028818_VPORT_Y_SCALE_ENA(1) | S_028818_VPORT_Y_OFFSET_ENA(1) |
588 S_028818_VPORT_Z_SCALE_ENA(1) | S_028818_VPORT_Z_OFFSET_ENA(1));
589
590 si_emit_compute(device, cs);
591 }
592
593 void
cik_create_gfx_config(struct radv_device * device)594 cik_create_gfx_config(struct radv_device *device)
595 {
596 struct radeon_cmdbuf *cs = device->ws->cs_create(device->ws, RING_GFX);
597 if (!cs)
598 return;
599
600 si_emit_graphics(device, cs);
601
602 while (cs->cdw & 7) {
603 if (device->physical_device->rad_info.gfx_ib_pad_with_type2)
604 radeon_emit(cs, PKT2_NOP_PAD);
605 else
606 radeon_emit(cs, PKT3_NOP_PAD);
607 }
608
609 device->gfx_init = device->ws->buffer_create(device->ws,
610 cs->cdw * 4, 4096,
611 RADEON_DOMAIN_GTT,
612 RADEON_FLAG_CPU_ACCESS|
613 RADEON_FLAG_NO_INTERPROCESS_SHARING |
614 RADEON_FLAG_READ_ONLY |
615 RADEON_FLAG_GTT_WC,
616 RADV_BO_PRIORITY_CS);
617 if (!device->gfx_init)
618 goto fail;
619
620 void *map = device->ws->buffer_map(device->gfx_init);
621 if (!map) {
622 device->ws->buffer_destroy(device->gfx_init);
623 device->gfx_init = NULL;
624 goto fail;
625 }
626 memcpy(map, cs->buf, cs->cdw * 4);
627
628 device->ws->buffer_unmap(device->gfx_init);
629 device->gfx_init_size_dw = cs->cdw;
630 fail:
631 device->ws->cs_destroy(cs);
632 }
633
634 static void
get_viewport_xform(const VkViewport * viewport,float scale[3],float translate[3])635 get_viewport_xform(const VkViewport *viewport,
636 float scale[3], float translate[3])
637 {
638 float x = viewport->x;
639 float y = viewport->y;
640 float half_width = 0.5f * viewport->width;
641 float half_height = 0.5f * viewport->height;
642 double n = viewport->minDepth;
643 double f = viewport->maxDepth;
644
645 scale[0] = half_width;
646 translate[0] = half_width + x;
647 scale[1] = half_height;
648 translate[1] = half_height + y;
649
650 scale[2] = (f - n);
651 translate[2] = n;
652 }
653
654 void
si_write_viewport(struct radeon_cmdbuf * cs,int first_vp,int count,const VkViewport * viewports)655 si_write_viewport(struct radeon_cmdbuf *cs, int first_vp,
656 int count, const VkViewport *viewports)
657 {
658 int i;
659
660 assert(count);
661 radeon_set_context_reg_seq(cs, R_02843C_PA_CL_VPORT_XSCALE +
662 first_vp * 4 * 6, count * 6);
663
664 for (i = 0; i < count; i++) {
665 float scale[3], translate[3];
666
667
668 get_viewport_xform(&viewports[i], scale, translate);
669 radeon_emit(cs, fui(scale[0]));
670 radeon_emit(cs, fui(translate[0]));
671 radeon_emit(cs, fui(scale[1]));
672 radeon_emit(cs, fui(translate[1]));
673 radeon_emit(cs, fui(scale[2]));
674 radeon_emit(cs, fui(translate[2]));
675 }
676
677 radeon_set_context_reg_seq(cs, R_0282D0_PA_SC_VPORT_ZMIN_0 +
678 first_vp * 4 * 2, count * 2);
679 for (i = 0; i < count; i++) {
680 float zmin = MIN2(viewports[i].minDepth, viewports[i].maxDepth);
681 float zmax = MAX2(viewports[i].minDepth, viewports[i].maxDepth);
682 radeon_emit(cs, fui(zmin));
683 radeon_emit(cs, fui(zmax));
684 }
685 }
686
si_scissor_from_viewport(const VkViewport * viewport)687 static VkRect2D si_scissor_from_viewport(const VkViewport *viewport)
688 {
689 float scale[3], translate[3];
690 VkRect2D rect;
691
692 get_viewport_xform(viewport, scale, translate);
693
694 rect.offset.x = translate[0] - fabsf(scale[0]);
695 rect.offset.y = translate[1] - fabsf(scale[1]);
696 rect.extent.width = ceilf(translate[0] + fabsf(scale[0])) - rect.offset.x;
697 rect.extent.height = ceilf(translate[1] + fabsf(scale[1])) - rect.offset.y;
698
699 return rect;
700 }
701
si_intersect_scissor(const VkRect2D * a,const VkRect2D * b)702 static VkRect2D si_intersect_scissor(const VkRect2D *a, const VkRect2D *b) {
703 VkRect2D ret;
704 ret.offset.x = MAX2(a->offset.x, b->offset.x);
705 ret.offset.y = MAX2(a->offset.y, b->offset.y);
706 ret.extent.width = MIN2(a->offset.x + a->extent.width,
707 b->offset.x + b->extent.width) - ret.offset.x;
708 ret.extent.height = MIN2(a->offset.y + a->extent.height,
709 b->offset.y + b->extent.height) - ret.offset.y;
710 return ret;
711 }
712
713 void
si_write_scissors(struct radeon_cmdbuf * cs,int first,int count,const VkRect2D * scissors,const VkViewport * viewports,bool can_use_guardband)714 si_write_scissors(struct radeon_cmdbuf *cs, int first,
715 int count, const VkRect2D *scissors,
716 const VkViewport *viewports, bool can_use_guardband)
717 {
718 int i;
719 float scale[3], translate[3], guardband_x = INFINITY, guardband_y = INFINITY;
720 const float max_range = 32767.0f;
721 if (!count)
722 return;
723
724 radeon_set_context_reg_seq(cs, R_028250_PA_SC_VPORT_SCISSOR_0_TL + first * 4 * 2, count * 2);
725 for (i = 0; i < count; i++) {
726 VkRect2D viewport_scissor = si_scissor_from_viewport(viewports + i);
727 VkRect2D scissor = si_intersect_scissor(&scissors[i], &viewport_scissor);
728
729 get_viewport_xform(viewports + i, scale, translate);
730 scale[0] = fabsf(scale[0]);
731 scale[1] = fabsf(scale[1]);
732
733 if (scale[0] < 0.5)
734 scale[0] = 0.5;
735 if (scale[1] < 0.5)
736 scale[1] = 0.5;
737
738 guardband_x = MIN2(guardband_x, (max_range - fabsf(translate[0])) / scale[0]);
739 guardband_y = MIN2(guardband_y, (max_range - fabsf(translate[1])) / scale[1]);
740
741 radeon_emit(cs, S_028250_TL_X(scissor.offset.x) |
742 S_028250_TL_Y(scissor.offset.y) |
743 S_028250_WINDOW_OFFSET_DISABLE(1));
744 radeon_emit(cs, S_028254_BR_X(scissor.offset.x + scissor.extent.width) |
745 S_028254_BR_Y(scissor.offset.y + scissor.extent.height));
746 }
747 if (!can_use_guardband) {
748 guardband_x = 1.0;
749 guardband_y = 1.0;
750 }
751
752 radeon_set_context_reg_seq(cs, R_028BE8_PA_CL_GB_VERT_CLIP_ADJ, 4);
753 radeon_emit(cs, fui(guardband_y));
754 radeon_emit(cs, fui(1.0));
755 radeon_emit(cs, fui(guardband_x));
756 radeon_emit(cs, fui(1.0));
757 }
758
759 static inline unsigned
radv_prims_for_vertices(struct radv_prim_vertex_count * info,unsigned num)760 radv_prims_for_vertices(struct radv_prim_vertex_count *info, unsigned num)
761 {
762 if (num == 0)
763 return 0;
764
765 if (info->incr == 0)
766 return 0;
767
768 if (num < info->min)
769 return 0;
770
771 return 1 + ((num - info->min) / info->incr);
772 }
773
774 static const struct radv_prim_vertex_count prim_size_table[] = {
775 [V_008958_DI_PT_NONE] = {0, 0},
776 [V_008958_DI_PT_POINTLIST] = {1, 1},
777 [V_008958_DI_PT_LINELIST] = {2, 2},
778 [V_008958_DI_PT_LINESTRIP] = {2, 1},
779 [V_008958_DI_PT_TRILIST] = {3, 3},
780 [V_008958_DI_PT_TRIFAN] = {3, 1},
781 [V_008958_DI_PT_TRISTRIP] = {3, 1},
782 [V_008958_DI_PT_LINELIST_ADJ] = {4, 4},
783 [V_008958_DI_PT_LINESTRIP_ADJ] = {4, 1},
784 [V_008958_DI_PT_TRILIST_ADJ] = {6, 6},
785 [V_008958_DI_PT_TRISTRIP_ADJ] = {6, 2},
786 [V_008958_DI_PT_RECTLIST] = {3, 3},
787 [V_008958_DI_PT_LINELOOP] = {2, 1},
788 [V_008958_DI_PT_POLYGON] = {3, 1},
789 [V_008958_DI_PT_2D_TRI_STRIP] = {0, 0},
790 };
791
792 uint32_t
si_get_ia_multi_vgt_param(struct radv_cmd_buffer * cmd_buffer,bool instanced_draw,bool indirect_draw,bool count_from_stream_output,uint32_t draw_vertex_count,unsigned topology)793 si_get_ia_multi_vgt_param(struct radv_cmd_buffer *cmd_buffer,
794 bool instanced_draw, bool indirect_draw,
795 bool count_from_stream_output,
796 uint32_t draw_vertex_count,
797 unsigned topology)
798 {
799 enum chip_class chip_class = cmd_buffer->device->physical_device->rad_info.chip_class;
800 enum radeon_family family = cmd_buffer->device->physical_device->rad_info.family;
801 struct radeon_info *info = &cmd_buffer->device->physical_device->rad_info;
802 const unsigned max_primgroup_in_wave = 2;
803 /* SWITCH_ON_EOP(0) is always preferable. */
804 bool wd_switch_on_eop = false;
805 bool ia_switch_on_eop = false;
806 bool ia_switch_on_eoi = false;
807 bool partial_vs_wave = false;
808 bool partial_es_wave = cmd_buffer->state.pipeline->graphics.ia_multi_vgt_param.partial_es_wave;
809 bool multi_instances_smaller_than_primgroup;
810 struct radv_prim_vertex_count prim_vertex_count = prim_size_table[topology];
811
812 if (radv_pipeline_has_tess(cmd_buffer->state.pipeline)) {
813 if (topology == V_008958_DI_PT_PATCH) {
814 prim_vertex_count.min = cmd_buffer->state.pipeline->graphics.tess_patch_control_points;
815 prim_vertex_count.incr = 1;
816 }
817 }
818
819 multi_instances_smaller_than_primgroup = indirect_draw;
820 if (!multi_instances_smaller_than_primgroup && instanced_draw) {
821 uint32_t num_prims = radv_prims_for_vertices(&prim_vertex_count, draw_vertex_count);
822 if (num_prims < cmd_buffer->state.pipeline->graphics.ia_multi_vgt_param.primgroup_size)
823 multi_instances_smaller_than_primgroup = true;
824 }
825
826 ia_switch_on_eoi = cmd_buffer->state.pipeline->graphics.ia_multi_vgt_param.ia_switch_on_eoi;
827 partial_vs_wave = cmd_buffer->state.pipeline->graphics.ia_multi_vgt_param.partial_vs_wave;
828
829 if (chip_class >= GFX7) {
830 /* WD_SWITCH_ON_EOP has no effect on GPUs with less than
831 * 4 shader engines. Set 1 to pass the assertion below.
832 * The other cases are hardware requirements. */
833 if (cmd_buffer->device->physical_device->rad_info.max_se < 4 ||
834 topology == V_008958_DI_PT_POLYGON ||
835 topology == V_008958_DI_PT_LINELOOP ||
836 topology == V_008958_DI_PT_TRIFAN ||
837 topology == V_008958_DI_PT_TRISTRIP_ADJ ||
838 (cmd_buffer->state.pipeline->graphics.prim_restart_enable &&
839 (cmd_buffer->device->physical_device->rad_info.family < CHIP_POLARIS10 ||
840 (topology != V_008958_DI_PT_POINTLIST &&
841 topology != V_008958_DI_PT_LINESTRIP))))
842 wd_switch_on_eop = true;
843
844 /* Hawaii hangs if instancing is enabled and WD_SWITCH_ON_EOP is 0.
845 * We don't know that for indirect drawing, so treat it as
846 * always problematic. */
847 if (family == CHIP_HAWAII &&
848 (instanced_draw || indirect_draw))
849 wd_switch_on_eop = true;
850
851 /* Performance recommendation for 4 SE Gfx7-8 parts if
852 * instances are smaller than a primgroup.
853 * Assume indirect draws always use small instances.
854 * This is needed for good VS wave utilization.
855 */
856 if (chip_class <= GFX8 &&
857 info->max_se == 4 &&
858 multi_instances_smaller_than_primgroup)
859 wd_switch_on_eop = true;
860
861 /* Required on GFX7 and later. */
862 if (info->max_se > 2 && !wd_switch_on_eop)
863 ia_switch_on_eoi = true;
864
865 /* Required by Hawaii and, for some special cases, by GFX8. */
866 if (ia_switch_on_eoi &&
867 (family == CHIP_HAWAII ||
868 (chip_class == GFX8 &&
869 /* max primgroup in wave is always 2 - leave this for documentation */
870 (radv_pipeline_has_gs(cmd_buffer->state.pipeline) || max_primgroup_in_wave != 2))))
871 partial_vs_wave = true;
872
873 /* Instancing bug on Bonaire. */
874 if (family == CHIP_BONAIRE && ia_switch_on_eoi &&
875 (instanced_draw || indirect_draw))
876 partial_vs_wave = true;
877
878 /* Hardware requirement when drawing primitives from a stream
879 * output buffer.
880 */
881 if (count_from_stream_output)
882 wd_switch_on_eop = true;
883
884 /* If the WD switch is false, the IA switch must be false too. */
885 assert(wd_switch_on_eop || !ia_switch_on_eop);
886 }
887 /* If SWITCH_ON_EOI is set, PARTIAL_ES_WAVE must be set too. */
888 if (chip_class <= GFX8 && ia_switch_on_eoi)
889 partial_es_wave = true;
890
891 if (radv_pipeline_has_gs(cmd_buffer->state.pipeline)) {
892 /* GS hw bug with single-primitive instances and SWITCH_ON_EOI.
893 * The hw doc says all multi-SE chips are affected, but amdgpu-pro Vulkan
894 * only applies it to Hawaii. Do what amdgpu-pro Vulkan does.
895 */
896 if (family == CHIP_HAWAII && ia_switch_on_eoi) {
897 bool set_vgt_flush = indirect_draw;
898 if (!set_vgt_flush && instanced_draw) {
899 uint32_t num_prims = radv_prims_for_vertices(&prim_vertex_count, draw_vertex_count);
900 if (num_prims <= 1)
901 set_vgt_flush = true;
902 }
903 if (set_vgt_flush)
904 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_VGT_FLUSH;
905 }
906 }
907
908 /* Workaround for a VGT hang when strip primitive types are used with
909 * primitive restart.
910 */
911 if (cmd_buffer->state.pipeline->graphics.prim_restart_enable &&
912 (topology == V_008958_DI_PT_LINESTRIP ||
913 topology == V_008958_DI_PT_TRISTRIP ||
914 topology == V_008958_DI_PT_LINESTRIP_ADJ ||
915 topology == V_008958_DI_PT_TRISTRIP_ADJ)) {
916 partial_vs_wave = true;
917 }
918
919 return cmd_buffer->state.pipeline->graphics.ia_multi_vgt_param.base |
920 S_028AA8_SWITCH_ON_EOP(ia_switch_on_eop) |
921 S_028AA8_SWITCH_ON_EOI(ia_switch_on_eoi) |
922 S_028AA8_PARTIAL_VS_WAVE_ON(partial_vs_wave) |
923 S_028AA8_PARTIAL_ES_WAVE_ON(partial_es_wave) |
924 S_028AA8_WD_SWITCH_ON_EOP(chip_class >= GFX7 ? wd_switch_on_eop : 0);
925
926 }
927
si_cs_emit_write_event_eop(struct radeon_cmdbuf * cs,enum chip_class chip_class,bool is_mec,unsigned event,unsigned event_flags,unsigned dst_sel,unsigned data_sel,uint64_t va,uint32_t new_fence,uint64_t gfx9_eop_bug_va)928 void si_cs_emit_write_event_eop(struct radeon_cmdbuf *cs,
929 enum chip_class chip_class,
930 bool is_mec,
931 unsigned event, unsigned event_flags,
932 unsigned dst_sel, unsigned data_sel,
933 uint64_t va,
934 uint32_t new_fence,
935 uint64_t gfx9_eop_bug_va)
936 {
937 unsigned op = EVENT_TYPE(event) |
938 EVENT_INDEX(event == V_028A90_CS_DONE ||
939 event == V_028A90_PS_DONE ? 6 : 5) |
940 event_flags;
941 unsigned is_gfx8_mec = is_mec && chip_class < GFX9;
942 unsigned sel = EOP_DST_SEL(dst_sel) |
943 EOP_DATA_SEL(data_sel);
944
945 /* Wait for write confirmation before writing data, but don't send
946 * an interrupt. */
947 if (data_sel != EOP_DATA_SEL_DISCARD)
948 sel |= EOP_INT_SEL(EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM);
949
950 if (chip_class >= GFX9 || is_gfx8_mec) {
951 /* A ZPASS_DONE or PIXEL_STAT_DUMP_EVENT (of the DB occlusion
952 * counters) must immediately precede every timestamp event to
953 * prevent a GPU hang on GFX9.
954 */
955 if (chip_class == GFX9 && !is_mec) {
956 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
957 radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_ZPASS_DONE) | EVENT_INDEX(1));
958 radeon_emit(cs, gfx9_eop_bug_va);
959 radeon_emit(cs, gfx9_eop_bug_va >> 32);
960 }
961
962 radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, is_gfx8_mec ? 5 : 6, false));
963 radeon_emit(cs, op);
964 radeon_emit(cs, sel);
965 radeon_emit(cs, va); /* address lo */
966 radeon_emit(cs, va >> 32); /* address hi */
967 radeon_emit(cs, new_fence); /* immediate data lo */
968 radeon_emit(cs, 0); /* immediate data hi */
969 if (!is_gfx8_mec)
970 radeon_emit(cs, 0); /* unused */
971 } else {
972 if (chip_class == GFX7 ||
973 chip_class == GFX8) {
974 /* Two EOP events are required to make all engines go idle
975 * (and optional cache flushes executed) before the timestamp
976 * is written.
977 */
978 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, false));
979 radeon_emit(cs, op);
980 radeon_emit(cs, va);
981 radeon_emit(cs, ((va >> 32) & 0xffff) | sel);
982 radeon_emit(cs, 0); /* immediate data */
983 radeon_emit(cs, 0); /* unused */
984 }
985
986 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, false));
987 radeon_emit(cs, op);
988 radeon_emit(cs, va);
989 radeon_emit(cs, ((va >> 32) & 0xffff) | sel);
990 radeon_emit(cs, new_fence); /* immediate data */
991 radeon_emit(cs, 0); /* unused */
992 }
993 }
994
995 void
radv_cp_wait_mem(struct radeon_cmdbuf * cs,uint32_t op,uint64_t va,uint32_t ref,uint32_t mask)996 radv_cp_wait_mem(struct radeon_cmdbuf *cs, uint32_t op, uint64_t va,
997 uint32_t ref, uint32_t mask)
998 {
999 assert(op == WAIT_REG_MEM_EQUAL ||
1000 op == WAIT_REG_MEM_NOT_EQUAL ||
1001 op == WAIT_REG_MEM_GREATER_OR_EQUAL);
1002
1003 radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, false));
1004 radeon_emit(cs, op | WAIT_REG_MEM_MEM_SPACE(1));
1005 radeon_emit(cs, va);
1006 radeon_emit(cs, va >> 32);
1007 radeon_emit(cs, ref); /* reference value */
1008 radeon_emit(cs, mask); /* mask */
1009 radeon_emit(cs, 4); /* poll interval */
1010 }
1011
1012 static void
si_emit_acquire_mem(struct radeon_cmdbuf * cs,bool is_mec,bool is_gfx9,unsigned cp_coher_cntl)1013 si_emit_acquire_mem(struct radeon_cmdbuf *cs,
1014 bool is_mec,
1015 bool is_gfx9,
1016 unsigned cp_coher_cntl)
1017 {
1018 if (is_mec || is_gfx9) {
1019 uint32_t hi_val = is_gfx9 ? 0xffffff : 0xff;
1020 radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 5, false) |
1021 PKT3_SHADER_TYPE_S(is_mec));
1022 radeon_emit(cs, cp_coher_cntl); /* CP_COHER_CNTL */
1023 radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */
1024 radeon_emit(cs, hi_val); /* CP_COHER_SIZE_HI */
1025 radeon_emit(cs, 0); /* CP_COHER_BASE */
1026 radeon_emit(cs, 0); /* CP_COHER_BASE_HI */
1027 radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */
1028 } else {
1029 /* ACQUIRE_MEM is only required on a compute ring. */
1030 radeon_emit(cs, PKT3(PKT3_SURFACE_SYNC, 3, false));
1031 radeon_emit(cs, cp_coher_cntl); /* CP_COHER_CNTL */
1032 radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */
1033 radeon_emit(cs, 0); /* CP_COHER_BASE */
1034 radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */
1035 }
1036 }
1037
1038 static void
gfx10_cs_emit_cache_flush(struct radeon_cmdbuf * cs,enum chip_class chip_class,uint32_t * flush_cnt,uint64_t flush_va,bool is_mec,enum radv_cmd_flush_bits flush_bits,enum rgp_flush_bits * sqtt_flush_bits,uint64_t gfx9_eop_bug_va)1039 gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs,
1040 enum chip_class chip_class,
1041 uint32_t *flush_cnt,
1042 uint64_t flush_va,
1043 bool is_mec,
1044 enum radv_cmd_flush_bits flush_bits,
1045 enum rgp_flush_bits *sqtt_flush_bits,
1046 uint64_t gfx9_eop_bug_va)
1047 {
1048 uint32_t gcr_cntl = 0;
1049 unsigned cb_db_event = 0;
1050
1051 /* We don't need these. */
1052 assert(!(flush_bits & (RADV_CMD_FLAG_VGT_STREAMOUT_SYNC)));
1053
1054 if (flush_bits & RADV_CMD_FLAG_INV_ICACHE) {
1055 gcr_cntl |= S_586_GLI_INV(V_586_GLI_ALL);
1056
1057 *sqtt_flush_bits |= RGP_FLUSH_INVAL_ICACHE;
1058 }
1059 if (flush_bits & RADV_CMD_FLAG_INV_SCACHE) {
1060 /* TODO: When writing to the SMEM L1 cache, we need to set SEQ
1061 * to FORWARD when both L1 and L2 are written out (WB or INV).
1062 */
1063 gcr_cntl |= S_586_GL1_INV(1) | S_586_GLK_INV(1);
1064
1065 *sqtt_flush_bits |= RGP_FLUSH_INVAL_SMEM_L0;
1066 }
1067 if (flush_bits & RADV_CMD_FLAG_INV_VCACHE) {
1068 gcr_cntl |= S_586_GL1_INV(1) | S_586_GLV_INV(1);
1069
1070 *sqtt_flush_bits |= RGP_FLUSH_INVAL_VMEM_L0 | RGP_FLUSH_INVAL_L1;
1071 }
1072 if (flush_bits & RADV_CMD_FLAG_INV_L2) {
1073 /* Writeback and invalidate everything in L2. */
1074 gcr_cntl |= S_586_GL2_INV(1) | S_586_GL2_WB(1) |
1075 S_586_GLM_INV(1) | S_586_GLM_WB(1);
1076
1077 *sqtt_flush_bits |= RGP_FLUSH_INVAL_L2;
1078 } else if (flush_bits & RADV_CMD_FLAG_WB_L2) {
1079 /* Writeback but do not invalidate.
1080 * GLM doesn't support WB alone. If WB is set, INV must be set too.
1081 */
1082 gcr_cntl |= S_586_GL2_WB(1) |
1083 S_586_GLM_WB(1) | S_586_GLM_INV(1);
1084
1085 *sqtt_flush_bits |= RGP_FLUSH_FLUSH_L2;
1086 }
1087
1088 /* TODO: Implement this new flag for GFX9+.
1089 else if (flush_bits & RADV_CMD_FLAG_INV_L2_METADATA)
1090 gcr_cntl |= S_586_GLM_INV(1) | S_586_GLM_WB(1);
1091 */
1092
1093 if (flush_bits & (RADV_CMD_FLAG_FLUSH_AND_INV_CB | RADV_CMD_FLAG_FLUSH_AND_INV_DB)) {
1094 /* TODO: trigger on RADV_CMD_FLAG_FLUSH_AND_INV_CB_META */
1095 if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_CB) {
1096 /* Flush CMASK/FMASK/DCC. Will wait for idle later. */
1097 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1098 radeon_emit(cs, EVENT_TYPE(V_028A90_FLUSH_AND_INV_CB_META) |
1099 EVENT_INDEX(0));
1100
1101 *sqtt_flush_bits |= RGP_FLUSH_FLUSH_CB | RGP_FLUSH_INVAL_CB;
1102 }
1103
1104 /* TODO: trigger on RADV_CMD_FLAG_FLUSH_AND_INV_DB_META ? */
1105 if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_DB) {
1106 /* Flush HTILE. Will wait for idle later. */
1107 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1108 radeon_emit(cs, EVENT_TYPE(V_028A90_FLUSH_AND_INV_DB_META) |
1109 EVENT_INDEX(0));
1110
1111 *sqtt_flush_bits |= RGP_FLUSH_FLUSH_DB | RGP_FLUSH_INVAL_DB;
1112 }
1113
1114 /* First flush CB/DB, then L1/L2. */
1115 gcr_cntl |= S_586_SEQ(V_586_SEQ_FORWARD);
1116
1117 if ((flush_bits & (RADV_CMD_FLAG_FLUSH_AND_INV_CB | RADV_CMD_FLAG_FLUSH_AND_INV_DB)) ==
1118 (RADV_CMD_FLAG_FLUSH_AND_INV_CB | RADV_CMD_FLAG_FLUSH_AND_INV_DB)) {
1119 cb_db_event = V_028A90_CACHE_FLUSH_AND_INV_TS_EVENT;
1120 } else if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_CB) {
1121 cb_db_event = V_028A90_FLUSH_AND_INV_CB_DATA_TS;
1122 } else if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_DB) {
1123 cb_db_event = V_028A90_FLUSH_AND_INV_DB_DATA_TS;
1124 } else {
1125 assert(0);
1126 }
1127 } else {
1128 /* Wait for graphics shaders to go idle if requested. */
1129 if (flush_bits & RADV_CMD_FLAG_PS_PARTIAL_FLUSH) {
1130 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1131 radeon_emit(cs, EVENT_TYPE(V_028A90_PS_PARTIAL_FLUSH) | EVENT_INDEX(4));
1132
1133 *sqtt_flush_bits |= RGP_FLUSH_PS_PARTIAL_FLUSH;
1134 } else if (flush_bits & RADV_CMD_FLAG_VS_PARTIAL_FLUSH) {
1135 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1136 radeon_emit(cs, EVENT_TYPE(V_028A90_VS_PARTIAL_FLUSH) | EVENT_INDEX(4));
1137
1138 *sqtt_flush_bits |= RGP_FLUSH_VS_PARTIAL_FLUSH;
1139 }
1140 }
1141
1142 if (flush_bits & RADV_CMD_FLAG_CS_PARTIAL_FLUSH) {
1143 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1144 radeon_emit(cs, EVENT_TYPE(V_028A90_CS_PARTIAL_FLUSH | EVENT_INDEX(4)));
1145
1146 *sqtt_flush_bits |= RGP_FLUSH_CS_PARTIAL_FLUSH;
1147 }
1148
1149 if (cb_db_event) {
1150 /* CB/DB flush and invalidate (or possibly just a wait for a
1151 * meta flush) via RELEASE_MEM.
1152 *
1153 * Combine this with other cache flushes when possible; this
1154 * requires affected shaders to be idle, so do it after the
1155 * CS_PARTIAL_FLUSH before (VS/PS partial flushes are always
1156 * implied).
1157 */
1158 /* Get GCR_CNTL fields, because the encoding is different in RELEASE_MEM. */
1159 unsigned glm_wb = G_586_GLM_WB(gcr_cntl);
1160 unsigned glm_inv = G_586_GLM_INV(gcr_cntl);
1161 unsigned glv_inv = G_586_GLV_INV(gcr_cntl);
1162 unsigned gl1_inv = G_586_GL1_INV(gcr_cntl);
1163 assert(G_586_GL2_US(gcr_cntl) == 0);
1164 assert(G_586_GL2_RANGE(gcr_cntl) == 0);
1165 assert(G_586_GL2_DISCARD(gcr_cntl) == 0);
1166 unsigned gl2_inv = G_586_GL2_INV(gcr_cntl);
1167 unsigned gl2_wb = G_586_GL2_WB(gcr_cntl);
1168 unsigned gcr_seq = G_586_SEQ(gcr_cntl);
1169
1170 gcr_cntl &= C_586_GLM_WB &
1171 C_586_GLM_INV &
1172 C_586_GLV_INV &
1173 C_586_GL1_INV &
1174 C_586_GL2_INV &
1175 C_586_GL2_WB; /* keep SEQ */
1176
1177 assert(flush_cnt);
1178 (*flush_cnt)++;
1179
1180 si_cs_emit_write_event_eop(cs, chip_class, false, cb_db_event,
1181 S_490_GLM_WB(glm_wb) |
1182 S_490_GLM_INV(glm_inv) |
1183 S_490_GLV_INV(glv_inv) |
1184 S_490_GL1_INV(gl1_inv) |
1185 S_490_GL2_INV(gl2_inv) |
1186 S_490_GL2_WB(gl2_wb) |
1187 S_490_SEQ(gcr_seq),
1188 EOP_DST_SEL_MEM,
1189 EOP_DATA_SEL_VALUE_32BIT,
1190 flush_va, *flush_cnt,
1191 gfx9_eop_bug_va);
1192
1193 radv_cp_wait_mem(cs, WAIT_REG_MEM_EQUAL, flush_va,
1194 *flush_cnt, 0xffffffff);
1195 }
1196
1197 /* VGT state sync */
1198 if (flush_bits & RADV_CMD_FLAG_VGT_FLUSH) {
1199 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1200 radeon_emit(cs, EVENT_TYPE(V_028A90_VGT_FLUSH) | EVENT_INDEX(0));
1201 }
1202
1203 /* Ignore fields that only modify the behavior of other fields. */
1204 if (gcr_cntl & C_586_GL1_RANGE & C_586_GL2_RANGE & C_586_SEQ) {
1205 /* Flush caches and wait for the caches to assert idle.
1206 * The cache flush is executed in the ME, but the PFP waits
1207 * for completion.
1208 */
1209 radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0));
1210 radeon_emit(cs, 0); /* CP_COHER_CNTL */
1211 radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */
1212 radeon_emit(cs, 0xffffff); /* CP_COHER_SIZE_HI */
1213 radeon_emit(cs, 0); /* CP_COHER_BASE */
1214 radeon_emit(cs, 0); /* CP_COHER_BASE_HI */
1215 radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */
1216 radeon_emit(cs, gcr_cntl); /* GCR_CNTL */
1217 } else if ((cb_db_event ||
1218 (flush_bits & (RADV_CMD_FLAG_VS_PARTIAL_FLUSH |
1219 RADV_CMD_FLAG_PS_PARTIAL_FLUSH |
1220 RADV_CMD_FLAG_CS_PARTIAL_FLUSH)))
1221 && !is_mec) {
1222 /* We need to ensure that PFP waits as well. */
1223 radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
1224 radeon_emit(cs, 0);
1225
1226 *sqtt_flush_bits |= RGP_FLUSH_PFP_SYNC_ME;
1227 }
1228
1229 if (flush_bits & RADV_CMD_FLAG_START_PIPELINE_STATS) {
1230 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1231 radeon_emit(cs, EVENT_TYPE(V_028A90_PIPELINESTAT_START) |
1232 EVENT_INDEX(0));
1233 } else if (flush_bits & RADV_CMD_FLAG_STOP_PIPELINE_STATS) {
1234 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1235 radeon_emit(cs, EVENT_TYPE(V_028A90_PIPELINESTAT_STOP) |
1236 EVENT_INDEX(0));
1237 }
1238 }
1239
1240 void
si_cs_emit_cache_flush(struct radeon_cmdbuf * cs,enum chip_class chip_class,uint32_t * flush_cnt,uint64_t flush_va,bool is_mec,enum radv_cmd_flush_bits flush_bits,enum rgp_flush_bits * sqtt_flush_bits,uint64_t gfx9_eop_bug_va)1241 si_cs_emit_cache_flush(struct radeon_cmdbuf *cs,
1242 enum chip_class chip_class,
1243 uint32_t *flush_cnt,
1244 uint64_t flush_va,
1245 bool is_mec,
1246 enum radv_cmd_flush_bits flush_bits,
1247 enum rgp_flush_bits *sqtt_flush_bits,
1248 uint64_t gfx9_eop_bug_va)
1249 {
1250 unsigned cp_coher_cntl = 0;
1251 uint32_t flush_cb_db = flush_bits & (RADV_CMD_FLAG_FLUSH_AND_INV_CB |
1252 RADV_CMD_FLAG_FLUSH_AND_INV_DB);
1253
1254 if (chip_class >= GFX10) {
1255 /* GFX10 cache flush handling is quite different. */
1256 gfx10_cs_emit_cache_flush(cs, chip_class, flush_cnt, flush_va,
1257 is_mec, flush_bits, sqtt_flush_bits,
1258 gfx9_eop_bug_va);
1259 return;
1260 }
1261
1262 if (flush_bits & RADV_CMD_FLAG_INV_ICACHE) {
1263 cp_coher_cntl |= S_0085F0_SH_ICACHE_ACTION_ENA(1);
1264 *sqtt_flush_bits |= RGP_FLUSH_INVAL_ICACHE;
1265 }
1266 if (flush_bits & RADV_CMD_FLAG_INV_SCACHE) {
1267 cp_coher_cntl |= S_0085F0_SH_KCACHE_ACTION_ENA(1);
1268 *sqtt_flush_bits |= RGP_FLUSH_INVAL_SMEM_L0;
1269 }
1270
1271 if (chip_class <= GFX8) {
1272 if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_CB) {
1273 cp_coher_cntl |= S_0085F0_CB_ACTION_ENA(1) |
1274 S_0085F0_CB0_DEST_BASE_ENA(1) |
1275 S_0085F0_CB1_DEST_BASE_ENA(1) |
1276 S_0085F0_CB2_DEST_BASE_ENA(1) |
1277 S_0085F0_CB3_DEST_BASE_ENA(1) |
1278 S_0085F0_CB4_DEST_BASE_ENA(1) |
1279 S_0085F0_CB5_DEST_BASE_ENA(1) |
1280 S_0085F0_CB6_DEST_BASE_ENA(1) |
1281 S_0085F0_CB7_DEST_BASE_ENA(1);
1282
1283 /* Necessary for DCC */
1284 if (chip_class >= GFX8) {
1285 si_cs_emit_write_event_eop(cs,
1286 chip_class,
1287 is_mec,
1288 V_028A90_FLUSH_AND_INV_CB_DATA_TS,
1289 0,
1290 EOP_DST_SEL_MEM,
1291 EOP_DATA_SEL_DISCARD,
1292 0, 0,
1293 gfx9_eop_bug_va);
1294 }
1295
1296 *sqtt_flush_bits |= RGP_FLUSH_FLUSH_CB | RGP_FLUSH_INVAL_CB;
1297 }
1298 if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_DB) {
1299 cp_coher_cntl |= S_0085F0_DB_ACTION_ENA(1) |
1300 S_0085F0_DB_DEST_BASE_ENA(1);
1301
1302 *sqtt_flush_bits |= RGP_FLUSH_FLUSH_DB | RGP_FLUSH_INVAL_DB;
1303 }
1304 }
1305
1306 if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_CB_META) {
1307 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1308 radeon_emit(cs, EVENT_TYPE(V_028A90_FLUSH_AND_INV_CB_META) | EVENT_INDEX(0));
1309
1310 *sqtt_flush_bits |= RGP_FLUSH_FLUSH_CB | RGP_FLUSH_INVAL_CB;
1311 }
1312
1313 if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_DB_META) {
1314 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1315 radeon_emit(cs, EVENT_TYPE(V_028A90_FLUSH_AND_INV_DB_META) | EVENT_INDEX(0));
1316
1317 *sqtt_flush_bits |= RGP_FLUSH_FLUSH_DB | RGP_FLUSH_INVAL_DB;
1318 }
1319
1320 if (flush_bits & RADV_CMD_FLAG_PS_PARTIAL_FLUSH) {
1321 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1322 radeon_emit(cs, EVENT_TYPE(V_028A90_PS_PARTIAL_FLUSH) | EVENT_INDEX(4));
1323
1324 *sqtt_flush_bits |= RGP_FLUSH_PS_PARTIAL_FLUSH;
1325 } else if (flush_bits & RADV_CMD_FLAG_VS_PARTIAL_FLUSH) {
1326 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1327 radeon_emit(cs, EVENT_TYPE(V_028A90_VS_PARTIAL_FLUSH) | EVENT_INDEX(4));
1328
1329 *sqtt_flush_bits |= RGP_FLUSH_VS_PARTIAL_FLUSH;
1330 }
1331
1332 if (flush_bits & RADV_CMD_FLAG_CS_PARTIAL_FLUSH) {
1333 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1334 radeon_emit(cs, EVENT_TYPE(V_028A90_CS_PARTIAL_FLUSH) | EVENT_INDEX(4));
1335
1336 *sqtt_flush_bits |= RGP_FLUSH_CS_PARTIAL_FLUSH;
1337 }
1338
1339 if (chip_class == GFX9 && flush_cb_db) {
1340 unsigned cb_db_event, tc_flags;
1341
1342 /* Set the CB/DB flush event. */
1343 cb_db_event = V_028A90_CACHE_FLUSH_AND_INV_TS_EVENT;
1344
1345 /* These are the only allowed combinations. If you need to
1346 * do multiple operations at once, do them separately.
1347 * All operations that invalidate L2 also seem to invalidate
1348 * metadata. Volatile (VOL) and WC flushes are not listed here.
1349 *
1350 * TC | TC_WB = writeback & invalidate L2 & L1
1351 * TC | TC_WB | TC_NC = writeback & invalidate L2 for MTYPE == NC
1352 * TC_WB | TC_NC = writeback L2 for MTYPE == NC
1353 * TC | TC_NC = invalidate L2 for MTYPE == NC
1354 * TC | TC_MD = writeback & invalidate L2 metadata (DCC, etc.)
1355 * TCL1 = invalidate L1
1356 */
1357 tc_flags = EVENT_TC_ACTION_ENA |
1358 EVENT_TC_MD_ACTION_ENA;
1359
1360 *sqtt_flush_bits |= RGP_FLUSH_FLUSH_CB | RGP_FLUSH_INVAL_CB |
1361 RGP_FLUSH_FLUSH_DB | RGP_FLUSH_INVAL_DB;
1362
1363 /* Ideally flush TC together with CB/DB. */
1364 if (flush_bits & RADV_CMD_FLAG_INV_L2) {
1365 /* Writeback and invalidate everything in L2 & L1. */
1366 tc_flags = EVENT_TC_ACTION_ENA |
1367 EVENT_TC_WB_ACTION_ENA;
1368
1369
1370 /* Clear the flags. */
1371 flush_bits &= ~(RADV_CMD_FLAG_INV_L2 |
1372 RADV_CMD_FLAG_WB_L2 |
1373 RADV_CMD_FLAG_INV_VCACHE);
1374
1375 *sqtt_flush_bits |= RGP_FLUSH_INVAL_L2;
1376 }
1377 assert(flush_cnt);
1378 (*flush_cnt)++;
1379
1380 si_cs_emit_write_event_eop(cs, chip_class, false, cb_db_event, tc_flags,
1381 EOP_DST_SEL_MEM,
1382 EOP_DATA_SEL_VALUE_32BIT,
1383 flush_va, *flush_cnt,
1384 gfx9_eop_bug_va);
1385 radv_cp_wait_mem(cs, WAIT_REG_MEM_EQUAL, flush_va,
1386 *flush_cnt, 0xffffffff);
1387 }
1388
1389 /* VGT state sync */
1390 if (flush_bits & RADV_CMD_FLAG_VGT_FLUSH) {
1391 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1392 radeon_emit(cs, EVENT_TYPE(V_028A90_VGT_FLUSH) | EVENT_INDEX(0));
1393 }
1394
1395 /* VGT streamout state sync */
1396 if (flush_bits & RADV_CMD_FLAG_VGT_STREAMOUT_SYNC) {
1397 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1398 radeon_emit(cs, EVENT_TYPE(V_028A90_VGT_STREAMOUT_SYNC) | EVENT_INDEX(0));
1399 }
1400
1401 /* Make sure ME is idle (it executes most packets) before continuing.
1402 * This prevents read-after-write hazards between PFP and ME.
1403 */
1404 if ((cp_coher_cntl ||
1405 (flush_bits & (RADV_CMD_FLAG_CS_PARTIAL_FLUSH |
1406 RADV_CMD_FLAG_INV_VCACHE |
1407 RADV_CMD_FLAG_INV_L2 |
1408 RADV_CMD_FLAG_WB_L2))) &&
1409 !is_mec) {
1410 radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
1411 radeon_emit(cs, 0);
1412
1413 *sqtt_flush_bits |= RGP_FLUSH_PFP_SYNC_ME;
1414 }
1415
1416 if ((flush_bits & RADV_CMD_FLAG_INV_L2) ||
1417 (chip_class <= GFX7 && (flush_bits & RADV_CMD_FLAG_WB_L2))) {
1418 si_emit_acquire_mem(cs, is_mec, chip_class == GFX9,
1419 cp_coher_cntl |
1420 S_0085F0_TC_ACTION_ENA(1) |
1421 S_0085F0_TCL1_ACTION_ENA(1) |
1422 S_0301F0_TC_WB_ACTION_ENA(chip_class >= GFX8));
1423 cp_coher_cntl = 0;
1424
1425 *sqtt_flush_bits |= RGP_FLUSH_INVAL_L2 | RGP_FLUSH_INVAL_VMEM_L0;
1426 } else {
1427 if(flush_bits & RADV_CMD_FLAG_WB_L2) {
1428 /* WB = write-back
1429 * NC = apply to non-coherent MTYPEs
1430 * (i.e. MTYPE <= 1, which is what we use everywhere)
1431 *
1432 * WB doesn't work without NC.
1433 */
1434 si_emit_acquire_mem(cs, is_mec,
1435 chip_class == GFX9,
1436 cp_coher_cntl |
1437 S_0301F0_TC_WB_ACTION_ENA(1) |
1438 S_0301F0_TC_NC_ACTION_ENA(1));
1439 cp_coher_cntl = 0;
1440
1441 *sqtt_flush_bits |= RGP_FLUSH_FLUSH_L2 | RGP_FLUSH_INVAL_VMEM_L0;
1442 }
1443 if (flush_bits & RADV_CMD_FLAG_INV_VCACHE) {
1444 si_emit_acquire_mem(cs, is_mec,
1445 chip_class == GFX9,
1446 cp_coher_cntl |
1447 S_0085F0_TCL1_ACTION_ENA(1));
1448 cp_coher_cntl = 0;
1449
1450 *sqtt_flush_bits |= RGP_FLUSH_INVAL_VMEM_L0;
1451 }
1452 }
1453
1454 /* When one of the DEST_BASE flags is set, SURFACE_SYNC waits for idle.
1455 * Therefore, it should be last. Done in PFP.
1456 */
1457 if (cp_coher_cntl)
1458 si_emit_acquire_mem(cs, is_mec, chip_class == GFX9, cp_coher_cntl);
1459
1460 if (flush_bits & RADV_CMD_FLAG_START_PIPELINE_STATS) {
1461 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1462 radeon_emit(cs, EVENT_TYPE(V_028A90_PIPELINESTAT_START) |
1463 EVENT_INDEX(0));
1464 } else if (flush_bits & RADV_CMD_FLAG_STOP_PIPELINE_STATS) {
1465 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1466 radeon_emit(cs, EVENT_TYPE(V_028A90_PIPELINESTAT_STOP) |
1467 EVENT_INDEX(0));
1468 }
1469 }
1470
1471 void
si_emit_cache_flush(struct radv_cmd_buffer * cmd_buffer)1472 si_emit_cache_flush(struct radv_cmd_buffer *cmd_buffer)
1473 {
1474 bool is_compute = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
1475
1476 if (is_compute)
1477 cmd_buffer->state.flush_bits &= ~(RADV_CMD_FLAG_FLUSH_AND_INV_CB |
1478 RADV_CMD_FLAG_FLUSH_AND_INV_CB_META |
1479 RADV_CMD_FLAG_FLUSH_AND_INV_DB |
1480 RADV_CMD_FLAG_FLUSH_AND_INV_DB_META |
1481 RADV_CMD_FLAG_PS_PARTIAL_FLUSH |
1482 RADV_CMD_FLAG_VS_PARTIAL_FLUSH |
1483 RADV_CMD_FLAG_VGT_FLUSH |
1484 RADV_CMD_FLAG_START_PIPELINE_STATS |
1485 RADV_CMD_FLAG_STOP_PIPELINE_STATS);
1486
1487 if (!cmd_buffer->state.flush_bits) {
1488 radv_describe_barrier_end_delayed(cmd_buffer);
1489 return;
1490 }
1491
1492 radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 128);
1493
1494 si_cs_emit_cache_flush(cmd_buffer->cs,
1495 cmd_buffer->device->physical_device->rad_info.chip_class,
1496 &cmd_buffer->gfx9_fence_idx,
1497 cmd_buffer->gfx9_fence_va,
1498 radv_cmd_buffer_uses_mec(cmd_buffer),
1499 cmd_buffer->state.flush_bits,
1500 &cmd_buffer->state.sqtt_flush_bits,
1501 cmd_buffer->gfx9_eop_bug_va);
1502
1503
1504 if (unlikely(cmd_buffer->device->trace_bo))
1505 radv_cmd_buffer_trace_emit(cmd_buffer);
1506
1507 /* Clear the caches that have been flushed to avoid syncing too much
1508 * when there is some pending active queries.
1509 */
1510 cmd_buffer->active_query_flush_bits &= ~cmd_buffer->state.flush_bits;
1511
1512 cmd_buffer->state.flush_bits = 0;
1513
1514 /* If the driver used a compute shader for resetting a query pool, it
1515 * should be finished at this point.
1516 */
1517 cmd_buffer->pending_reset_query = false;
1518
1519 radv_describe_barrier_end_delayed(cmd_buffer);
1520 }
1521
1522 /* sets the CP predication state using a boolean stored at va */
1523 void
si_emit_set_predication_state(struct radv_cmd_buffer * cmd_buffer,bool draw_visible,uint64_t va)1524 si_emit_set_predication_state(struct radv_cmd_buffer *cmd_buffer,
1525 bool draw_visible, uint64_t va)
1526 {
1527 uint32_t op = 0;
1528
1529 if (va) {
1530 op = PRED_OP(PREDICATION_OP_BOOL64);
1531
1532 /* PREDICATION_DRAW_VISIBLE means that if the 32-bit value is
1533 * zero, all rendering commands are discarded. Otherwise, they
1534 * are discarded if the value is non zero.
1535 */
1536 op |= draw_visible ? PREDICATION_DRAW_VISIBLE :
1537 PREDICATION_DRAW_NOT_VISIBLE;
1538 }
1539 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9) {
1540 radeon_emit(cmd_buffer->cs, PKT3(PKT3_SET_PREDICATION, 2, 0));
1541 radeon_emit(cmd_buffer->cs, op);
1542 radeon_emit(cmd_buffer->cs, va);
1543 radeon_emit(cmd_buffer->cs, va >> 32);
1544 } else {
1545 radeon_emit(cmd_buffer->cs, PKT3(PKT3_SET_PREDICATION, 1, 0));
1546 radeon_emit(cmd_buffer->cs, va);
1547 radeon_emit(cmd_buffer->cs, op | ((va >> 32) & 0xFF));
1548 }
1549 }
1550
1551 /* Set this if you want the 3D engine to wait until CP DMA is done.
1552 * It should be set on the last CP DMA packet. */
1553 #define CP_DMA_SYNC (1 << 0)
1554
1555 /* Set this if the source data was used as a destination in a previous CP DMA
1556 * packet. It's for preventing a read-after-write (RAW) hazard between two
1557 * CP DMA packets. */
1558 #define CP_DMA_RAW_WAIT (1 << 1)
1559 #define CP_DMA_USE_L2 (1 << 2)
1560 #define CP_DMA_CLEAR (1 << 3)
1561
1562 /* Alignment for optimal performance. */
1563 #define SI_CPDMA_ALIGNMENT 32
1564
1565 /* The max number of bytes that can be copied per packet. */
cp_dma_max_byte_count(struct radv_cmd_buffer * cmd_buffer)1566 static inline unsigned cp_dma_max_byte_count(struct radv_cmd_buffer *cmd_buffer)
1567 {
1568 unsigned max = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9 ?
1569 S_414_BYTE_COUNT_GFX9(~0u) :
1570 S_414_BYTE_COUNT_GFX6(~0u);
1571
1572 /* make it aligned for optimal performance */
1573 return max & ~(SI_CPDMA_ALIGNMENT - 1);
1574 }
1575
1576 /* Emit a CP DMA packet to do a copy from one buffer to another, or to clear
1577 * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit
1578 * clear value.
1579 */
si_emit_cp_dma(struct radv_cmd_buffer * cmd_buffer,uint64_t dst_va,uint64_t src_va,unsigned size,unsigned flags)1580 static void si_emit_cp_dma(struct radv_cmd_buffer *cmd_buffer,
1581 uint64_t dst_va, uint64_t src_va,
1582 unsigned size, unsigned flags)
1583 {
1584 struct radeon_cmdbuf *cs = cmd_buffer->cs;
1585 uint32_t header = 0, command = 0;
1586
1587 assert(size <= cp_dma_max_byte_count(cmd_buffer));
1588
1589 radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 9);
1590 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9)
1591 command |= S_414_BYTE_COUNT_GFX9(size);
1592 else
1593 command |= S_414_BYTE_COUNT_GFX6(size);
1594
1595 /* Sync flags. */
1596 if (flags & CP_DMA_SYNC)
1597 header |= S_411_CP_SYNC(1);
1598 else {
1599 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9)
1600 command |= S_414_DISABLE_WR_CONFIRM_GFX9(1);
1601 else
1602 command |= S_414_DISABLE_WR_CONFIRM_GFX6(1);
1603 }
1604
1605 if (flags & CP_DMA_RAW_WAIT)
1606 command |= S_414_RAW_WAIT(1);
1607
1608 /* Src and dst flags. */
1609 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9 &&
1610 !(flags & CP_DMA_CLEAR) &&
1611 src_va == dst_va)
1612 header |= S_411_DST_SEL(V_411_NOWHERE); /* prefetch only */
1613 else if (flags & CP_DMA_USE_L2)
1614 header |= S_411_DST_SEL(V_411_DST_ADDR_TC_L2);
1615
1616 if (flags & CP_DMA_CLEAR)
1617 header |= S_411_SRC_SEL(V_411_DATA);
1618 else if (flags & CP_DMA_USE_L2)
1619 header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2);
1620
1621 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7) {
1622 radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, cmd_buffer->state.predicating));
1623 radeon_emit(cs, header);
1624 radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */
1625 radeon_emit(cs, src_va >> 32); /* SRC_ADDR_HI [31:0] */
1626 radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */
1627 radeon_emit(cs, dst_va >> 32); /* DST_ADDR_HI [31:0] */
1628 radeon_emit(cs, command);
1629 } else {
1630 assert(!(flags & CP_DMA_USE_L2));
1631 header |= S_411_SRC_ADDR_HI(src_va >> 32);
1632 radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, cmd_buffer->state.predicating));
1633 radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */
1634 radeon_emit(cs, header); /* SRC_ADDR_HI [15:0] + flags. */
1635 radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */
1636 radeon_emit(cs, (dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
1637 radeon_emit(cs, command);
1638 }
1639
1640 /* CP DMA is executed in ME, but index buffers are read by PFP.
1641 * This ensures that ME (CP DMA) is idle before PFP starts fetching
1642 * indices. If we wanted to execute CP DMA in PFP, this packet
1643 * should precede it.
1644 */
1645 if (flags & CP_DMA_SYNC) {
1646 if (cmd_buffer->queue_family_index == RADV_QUEUE_GENERAL) {
1647 radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating));
1648 radeon_emit(cs, 0);
1649 }
1650
1651 /* CP will see the sync flag and wait for all DMAs to complete. */
1652 cmd_buffer->state.dma_is_busy = false;
1653 }
1654
1655 if (unlikely(cmd_buffer->device->trace_bo))
1656 radv_cmd_buffer_trace_emit(cmd_buffer);
1657 }
1658
si_cp_dma_prefetch(struct radv_cmd_buffer * cmd_buffer,uint64_t va,unsigned size)1659 void si_cp_dma_prefetch(struct radv_cmd_buffer *cmd_buffer, uint64_t va,
1660 unsigned size)
1661 {
1662 uint64_t aligned_va = va & ~(SI_CPDMA_ALIGNMENT - 1);
1663 uint64_t aligned_size = ((va + size + SI_CPDMA_ALIGNMENT -1) & ~(SI_CPDMA_ALIGNMENT - 1)) - aligned_va;
1664
1665 si_emit_cp_dma(cmd_buffer, aligned_va, aligned_va,
1666 aligned_size, CP_DMA_USE_L2);
1667 }
1668
si_cp_dma_prepare(struct radv_cmd_buffer * cmd_buffer,uint64_t byte_count,uint64_t remaining_size,unsigned * flags)1669 static void si_cp_dma_prepare(struct radv_cmd_buffer *cmd_buffer, uint64_t byte_count,
1670 uint64_t remaining_size, unsigned *flags)
1671 {
1672
1673 /* Flush the caches for the first copy only.
1674 * Also wait for the previous CP DMA operations.
1675 */
1676 if (cmd_buffer->state.flush_bits) {
1677 si_emit_cache_flush(cmd_buffer);
1678 *flags |= CP_DMA_RAW_WAIT;
1679 }
1680
1681 /* Do the synchronization after the last dma, so that all data
1682 * is written to memory.
1683 */
1684 if (byte_count == remaining_size)
1685 *flags |= CP_DMA_SYNC;
1686 }
1687
si_cp_dma_realign_engine(struct radv_cmd_buffer * cmd_buffer,unsigned size)1688 static void si_cp_dma_realign_engine(struct radv_cmd_buffer *cmd_buffer, unsigned size)
1689 {
1690 uint64_t va;
1691 uint32_t offset;
1692 unsigned dma_flags = 0;
1693 unsigned buf_size = SI_CPDMA_ALIGNMENT * 2;
1694 void *ptr;
1695
1696 assert(size < SI_CPDMA_ALIGNMENT);
1697
1698 radv_cmd_buffer_upload_alloc(cmd_buffer, buf_size, SI_CPDMA_ALIGNMENT, &offset, &ptr);
1699
1700 va = radv_buffer_get_va(cmd_buffer->upload.upload_bo);
1701 va += offset;
1702
1703 si_cp_dma_prepare(cmd_buffer, size, size, &dma_flags);
1704
1705 si_emit_cp_dma(cmd_buffer, va, va + SI_CPDMA_ALIGNMENT, size,
1706 dma_flags);
1707 }
1708
si_cp_dma_buffer_copy(struct radv_cmd_buffer * cmd_buffer,uint64_t src_va,uint64_t dest_va,uint64_t size)1709 void si_cp_dma_buffer_copy(struct radv_cmd_buffer *cmd_buffer,
1710 uint64_t src_va, uint64_t dest_va,
1711 uint64_t size)
1712 {
1713 uint64_t main_src_va, main_dest_va;
1714 uint64_t skipped_size = 0, realign_size = 0;
1715
1716 /* Assume that we are not going to sync after the last DMA operation. */
1717 cmd_buffer->state.dma_is_busy = true;
1718
1719 if (cmd_buffer->device->physical_device->rad_info.family <= CHIP_CARRIZO ||
1720 cmd_buffer->device->physical_device->rad_info.family == CHIP_STONEY) {
1721 /* If the size is not aligned, we must add a dummy copy at the end
1722 * just to align the internal counter. Otherwise, the DMA engine
1723 * would slow down by an order of magnitude for following copies.
1724 */
1725 if (size % SI_CPDMA_ALIGNMENT)
1726 realign_size = SI_CPDMA_ALIGNMENT - (size % SI_CPDMA_ALIGNMENT);
1727
1728 /* If the copy begins unaligned, we must start copying from the next
1729 * aligned block and the skipped part should be copied after everything
1730 * else has been copied. Only the src alignment matters, not dst.
1731 */
1732 if (src_va % SI_CPDMA_ALIGNMENT) {
1733 skipped_size = SI_CPDMA_ALIGNMENT - (src_va % SI_CPDMA_ALIGNMENT);
1734 /* The main part will be skipped if the size is too small. */
1735 skipped_size = MIN2(skipped_size, size);
1736 size -= skipped_size;
1737 }
1738 }
1739 main_src_va = src_va + skipped_size;
1740 main_dest_va = dest_va + skipped_size;
1741
1742 while (size) {
1743 unsigned dma_flags = 0;
1744 unsigned byte_count = MIN2(size, cp_dma_max_byte_count(cmd_buffer));
1745
1746 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) {
1747 /* DMA operations via L2 are coherent and faster.
1748 * TODO: GFX7-GFX9 should also support this but it
1749 * requires tests/benchmarks.
1750 */
1751 dma_flags |= CP_DMA_USE_L2;
1752 }
1753
1754 si_cp_dma_prepare(cmd_buffer, byte_count,
1755 size + skipped_size + realign_size,
1756 &dma_flags);
1757
1758 dma_flags &= ~CP_DMA_SYNC;
1759
1760 si_emit_cp_dma(cmd_buffer, main_dest_va, main_src_va,
1761 byte_count, dma_flags);
1762
1763 size -= byte_count;
1764 main_src_va += byte_count;
1765 main_dest_va += byte_count;
1766 }
1767
1768 if (skipped_size) {
1769 unsigned dma_flags = 0;
1770
1771 si_cp_dma_prepare(cmd_buffer, skipped_size,
1772 size + skipped_size + realign_size,
1773 &dma_flags);
1774
1775 si_emit_cp_dma(cmd_buffer, dest_va, src_va,
1776 skipped_size, dma_flags);
1777 }
1778 if (realign_size)
1779 si_cp_dma_realign_engine(cmd_buffer, realign_size);
1780 }
1781
si_cp_dma_clear_buffer(struct radv_cmd_buffer * cmd_buffer,uint64_t va,uint64_t size,unsigned value)1782 void si_cp_dma_clear_buffer(struct radv_cmd_buffer *cmd_buffer, uint64_t va,
1783 uint64_t size, unsigned value)
1784 {
1785
1786 if (!size)
1787 return;
1788
1789 assert(va % 4 == 0 && size % 4 == 0);
1790
1791 /* Assume that we are not going to sync after the last DMA operation. */
1792 cmd_buffer->state.dma_is_busy = true;
1793
1794 while (size) {
1795 unsigned byte_count = MIN2(size, cp_dma_max_byte_count(cmd_buffer));
1796 unsigned dma_flags = CP_DMA_CLEAR;
1797
1798 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) {
1799 /* DMA operations via L2 are coherent and faster.
1800 * TODO: GFX7-GFX9 should also support this but it
1801 * requires tests/benchmarks.
1802 */
1803 dma_flags |= CP_DMA_USE_L2;
1804 }
1805
1806 si_cp_dma_prepare(cmd_buffer, byte_count, size, &dma_flags);
1807
1808 /* Emit the clear packet. */
1809 si_emit_cp_dma(cmd_buffer, va, value, byte_count,
1810 dma_flags);
1811
1812 size -= byte_count;
1813 va += byte_count;
1814 }
1815 }
1816
si_cp_dma_wait_for_idle(struct radv_cmd_buffer * cmd_buffer)1817 void si_cp_dma_wait_for_idle(struct radv_cmd_buffer *cmd_buffer)
1818 {
1819 if (cmd_buffer->device->physical_device->rad_info.chip_class < GFX7)
1820 return;
1821
1822 if (!cmd_buffer->state.dma_is_busy)
1823 return;
1824
1825 /* Issue a dummy DMA that copies zero bytes.
1826 *
1827 * The DMA engine will see that there's no work to do and skip this
1828 * DMA request, however, the CP will see the sync flag and still wait
1829 * for all DMAs to complete.
1830 */
1831 si_emit_cp_dma(cmd_buffer, 0, 0, 0, CP_DMA_SYNC);
1832
1833 cmd_buffer->state.dma_is_busy = false;
1834 }
1835
1836 /* For MSAA sample positions. */
1837 #define FILL_SREG(s0x, s0y, s1x, s1y, s2x, s2y, s3x, s3y) \
1838 ((((unsigned)(s0x) & 0xf) << 0) | (((unsigned)(s0y) & 0xf) << 4) | \
1839 (((unsigned)(s1x) & 0xf) << 8) | (((unsigned)(s1y) & 0xf) << 12) | \
1840 (((unsigned)(s2x) & 0xf) << 16) | (((unsigned)(s2y) & 0xf) << 20) | \
1841 (((unsigned)(s3x) & 0xf) << 24) | (((unsigned)(s3y) & 0xf) << 28))
1842
1843 /* For obtaining location coordinates from registers */
1844 #define SEXT4(x) ((int)((x) | ((x) & 0x8 ? 0xfffffff0 : 0)))
1845 #define GET_SFIELD(reg, index) SEXT4(((reg) >> ((index) * 4)) & 0xf)
1846 #define GET_SX(reg, index) GET_SFIELD((reg)[(index) / 4], ((index) % 4) * 2)
1847 #define GET_SY(reg, index) GET_SFIELD((reg)[(index) / 4], ((index) % 4) * 2 + 1)
1848
1849 /* 1x MSAA */
1850 static const uint32_t sample_locs_1x =
1851 FILL_SREG(0, 0, 0, 0, 0, 0, 0, 0);
1852 static const unsigned max_dist_1x = 0;
1853 static const uint64_t centroid_priority_1x = 0x0000000000000000ull;
1854
1855 /* 2xMSAA */
1856 static const uint32_t sample_locs_2x =
1857 FILL_SREG(4,4, -4, -4, 0, 0, 0, 0);
1858 static const unsigned max_dist_2x = 4;
1859 static const uint64_t centroid_priority_2x = 0x1010101010101010ull;
1860
1861 /* 4xMSAA */
1862 static const uint32_t sample_locs_4x =
1863 FILL_SREG(-2,-6, 6, -2, -6, 2, 2, 6);
1864 static const unsigned max_dist_4x = 6;
1865 static const uint64_t centroid_priority_4x = 0x3210321032103210ull;
1866
1867 /* 8xMSAA */
1868 static const uint32_t sample_locs_8x[] = {
1869 FILL_SREG( 1,-3, -1, 3, 5, 1, -3,-5),
1870 FILL_SREG(-5, 5, -7,-1, 3, 7, 7,-7),
1871 /* The following are unused by hardware, but we emit them to IBs
1872 * instead of multiple SET_CONTEXT_REG packets. */
1873 0,
1874 0,
1875 };
1876 static const unsigned max_dist_8x = 7;
1877 static const uint64_t centroid_priority_8x = 0x7654321076543210ull;
1878
radv_get_default_max_sample_dist(int log_samples)1879 unsigned radv_get_default_max_sample_dist(int log_samples)
1880 {
1881 unsigned max_dist[] = {
1882 max_dist_1x,
1883 max_dist_2x,
1884 max_dist_4x,
1885 max_dist_8x,
1886 };
1887 return max_dist[log_samples];
1888 }
1889
radv_emit_default_sample_locations(struct radeon_cmdbuf * cs,int nr_samples)1890 void radv_emit_default_sample_locations(struct radeon_cmdbuf *cs, int nr_samples)
1891 {
1892 switch (nr_samples) {
1893 default:
1894 case 1:
1895 radeon_set_context_reg_seq(cs, R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2);
1896 radeon_emit(cs, (uint32_t)centroid_priority_1x);
1897 radeon_emit(cs, centroid_priority_1x >> 32);
1898 radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_1x);
1899 radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_1x);
1900 radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_1x);
1901 radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_1x);
1902 break;
1903 case 2:
1904 radeon_set_context_reg_seq(cs, R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2);
1905 radeon_emit(cs, (uint32_t)centroid_priority_2x);
1906 radeon_emit(cs, centroid_priority_2x >> 32);
1907 radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_2x);
1908 radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_2x);
1909 radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_2x);
1910 radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_2x);
1911 break;
1912 case 4:
1913 radeon_set_context_reg_seq(cs, R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2);
1914 radeon_emit(cs, (uint32_t)centroid_priority_4x);
1915 radeon_emit(cs, centroid_priority_4x >> 32);
1916 radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_4x);
1917 radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_4x);
1918 radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_4x);
1919 radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_4x);
1920 break;
1921 case 8:
1922 radeon_set_context_reg_seq(cs, R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2);
1923 radeon_emit(cs, (uint32_t)centroid_priority_8x);
1924 radeon_emit(cs, centroid_priority_8x >> 32);
1925 radeon_set_context_reg_seq(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, 14);
1926 radeon_emit_array(cs, sample_locs_8x, 4);
1927 radeon_emit_array(cs, sample_locs_8x, 4);
1928 radeon_emit_array(cs, sample_locs_8x, 4);
1929 radeon_emit_array(cs, sample_locs_8x, 2);
1930 break;
1931 }
1932 }
1933
radv_get_sample_position(struct radv_device * device,unsigned sample_count,unsigned sample_index,float * out_value)1934 static void radv_get_sample_position(struct radv_device *device,
1935 unsigned sample_count,
1936 unsigned sample_index, float *out_value)
1937 {
1938 const uint32_t *sample_locs;
1939
1940 switch (sample_count) {
1941 case 1:
1942 default:
1943 sample_locs = &sample_locs_1x;
1944 break;
1945 case 2:
1946 sample_locs = &sample_locs_2x;
1947 break;
1948 case 4:
1949 sample_locs = &sample_locs_4x;
1950 break;
1951 case 8:
1952 sample_locs = sample_locs_8x;
1953 break;
1954 }
1955
1956 out_value[0] = (GET_SX(sample_locs, sample_index) + 8) / 16.0f;
1957 out_value[1] = (GET_SY(sample_locs, sample_index) + 8) / 16.0f;
1958 }
1959
radv_device_init_msaa(struct radv_device * device)1960 void radv_device_init_msaa(struct radv_device *device)
1961 {
1962 int i;
1963
1964 radv_get_sample_position(device, 1, 0, device->sample_locations_1x[0]);
1965
1966 for (i = 0; i < 2; i++)
1967 radv_get_sample_position(device, 2, i, device->sample_locations_2x[i]);
1968 for (i = 0; i < 4; i++)
1969 radv_get_sample_position(device, 4, i, device->sample_locations_4x[i]);
1970 for (i = 0; i < 8; i++)
1971 radv_get_sample_position(device, 8, i, device->sample_locations_8x[i]);
1972 }
1973