1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_private.h"
25
26 static bool
lookup_blorp_shader(struct blorp_batch * batch,const void * key,uint32_t key_size,uint32_t * kernel_out,void * prog_data_out)27 lookup_blorp_shader(struct blorp_batch *batch,
28 const void *key, uint32_t key_size,
29 uint32_t *kernel_out, void *prog_data_out)
30 {
31 struct blorp_context *blorp = batch->blorp;
32 struct anv_device *device = blorp->driver_ctx;
33
34 /* The default cache must be a real cache */
35 assert(device->default_pipeline_cache.cache);
36
37 struct anv_shader_bin *bin =
38 anv_pipeline_cache_search(&device->default_pipeline_cache, key, key_size);
39 if (!bin)
40 return false;
41
42 /* The cache already has a reference and it's not going anywhere so there
43 * is no need to hold a second reference.
44 */
45 anv_shader_bin_unref(device, bin);
46
47 *kernel_out = bin->kernel.offset;
48 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
49
50 return true;
51 }
52
53 static bool
upload_blorp_shader(struct blorp_batch * batch,uint32_t stage,const void * key,uint32_t key_size,const void * kernel,uint32_t kernel_size,const struct brw_stage_prog_data * prog_data,uint32_t prog_data_size,uint32_t * kernel_out,void * prog_data_out)54 upload_blorp_shader(struct blorp_batch *batch, uint32_t stage,
55 const void *key, uint32_t key_size,
56 const void *kernel, uint32_t kernel_size,
57 const struct brw_stage_prog_data *prog_data,
58 uint32_t prog_data_size,
59 uint32_t *kernel_out, void *prog_data_out)
60 {
61 struct blorp_context *blorp = batch->blorp;
62 struct anv_device *device = blorp->driver_ctx;
63
64 /* The blorp cache must be a real cache */
65 assert(device->default_pipeline_cache.cache);
66
67 struct anv_pipeline_bind_map bind_map = {
68 .surface_count = 0,
69 .sampler_count = 0,
70 };
71
72 struct anv_shader_bin *bin =
73 anv_pipeline_cache_upload_kernel(&device->default_pipeline_cache, stage,
74 key, key_size, kernel, kernel_size,
75 prog_data, prog_data_size,
76 NULL, 0, NULL, &bind_map);
77
78 if (!bin)
79 return false;
80
81 /* The cache already has a reference and it's not going anywhere so there
82 * is no need to hold a second reference.
83 */
84 anv_shader_bin_unref(device, bin);
85
86 *kernel_out = bin->kernel.offset;
87 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
88
89 return true;
90 }
91
92 void
anv_device_init_blorp(struct anv_device * device)93 anv_device_init_blorp(struct anv_device *device)
94 {
95 blorp_init(&device->blorp, device, &device->isl_dev);
96 device->blorp.compiler = device->physical->compiler;
97 device->blorp.lookup_shader = lookup_blorp_shader;
98 device->blorp.upload_shader = upload_blorp_shader;
99 switch (device->info.gen) {
100 case 7:
101 if (device->info.is_haswell) {
102 device->blorp.exec = gen75_blorp_exec;
103 } else {
104 device->blorp.exec = gen7_blorp_exec;
105 }
106 break;
107 case 8:
108 device->blorp.exec = gen8_blorp_exec;
109 break;
110 case 9:
111 device->blorp.exec = gen9_blorp_exec;
112 break;
113 case 11:
114 device->blorp.exec = gen11_blorp_exec;
115 break;
116 case 12:
117 device->blorp.exec = gen12_blorp_exec;
118 break;
119 default:
120 unreachable("Unknown hardware generation");
121 }
122 }
123
124 void
anv_device_finish_blorp(struct anv_device * device)125 anv_device_finish_blorp(struct anv_device *device)
126 {
127 blorp_finish(&device->blorp);
128 }
129
130 static void
get_blorp_surf_for_anv_buffer(struct anv_device * device,struct anv_buffer * buffer,uint64_t offset,uint32_t width,uint32_t height,uint32_t row_pitch,enum isl_format format,bool is_dest,struct blorp_surf * blorp_surf,struct isl_surf * isl_surf)131 get_blorp_surf_for_anv_buffer(struct anv_device *device,
132 struct anv_buffer *buffer, uint64_t offset,
133 uint32_t width, uint32_t height,
134 uint32_t row_pitch, enum isl_format format,
135 bool is_dest,
136 struct blorp_surf *blorp_surf,
137 struct isl_surf *isl_surf)
138 {
139 const struct isl_format_layout *fmtl =
140 isl_format_get_layout(format);
141 bool ok UNUSED;
142
143 /* ASTC is the only format which doesn't support linear layouts.
144 * Create an equivalently sized surface with ISL to get around this.
145 */
146 if (fmtl->txc == ISL_TXC_ASTC) {
147 /* Use an equivalently sized format */
148 format = ISL_FORMAT_R32G32B32A32_UINT;
149 assert(fmtl->bpb == isl_format_get_layout(format)->bpb);
150
151 /* Shrink the dimensions for the new format */
152 width = DIV_ROUND_UP(width, fmtl->bw);
153 height = DIV_ROUND_UP(height, fmtl->bh);
154 }
155
156 *blorp_surf = (struct blorp_surf) {
157 .surf = isl_surf,
158 .addr = {
159 .buffer = buffer->address.bo,
160 .offset = buffer->address.offset + offset,
161 .mocs = anv_mocs(device, buffer->address.bo,
162 is_dest ? ISL_SURF_USAGE_RENDER_TARGET_BIT
163 : ISL_SURF_USAGE_TEXTURE_BIT),
164 },
165 };
166
167 ok = isl_surf_init(&device->isl_dev, isl_surf,
168 .dim = ISL_SURF_DIM_2D,
169 .format = format,
170 .width = width,
171 .height = height,
172 .depth = 1,
173 .levels = 1,
174 .array_len = 1,
175 .samples = 1,
176 .row_pitch_B = row_pitch,
177 .usage = is_dest ? ISL_SURF_USAGE_RENDER_TARGET_BIT
178 : ISL_SURF_USAGE_TEXTURE_BIT,
179 .tiling_flags = ISL_TILING_LINEAR_BIT);
180 assert(ok);
181 }
182
183 /* Pick something high enough that it won't be used in core and low enough it
184 * will never map to an extension.
185 */
186 #define ANV_IMAGE_LAYOUT_EXPLICIT_AUX (VkImageLayout)10000000
187
188 static struct blorp_address
anv_to_blorp_address(struct anv_address addr)189 anv_to_blorp_address(struct anv_address addr)
190 {
191 return (struct blorp_address) {
192 .buffer = addr.bo,
193 .offset = addr.offset,
194 };
195 }
196
197 static void
get_blorp_surf_for_anv_image(const struct anv_device * device,const struct anv_image * image,VkImageAspectFlags aspect,VkImageUsageFlags usage,VkImageLayout layout,enum isl_aux_usage aux_usage,struct blorp_surf * blorp_surf)198 get_blorp_surf_for_anv_image(const struct anv_device *device,
199 const struct anv_image *image,
200 VkImageAspectFlags aspect,
201 VkImageUsageFlags usage,
202 VkImageLayout layout,
203 enum isl_aux_usage aux_usage,
204 struct blorp_surf *blorp_surf)
205 {
206 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
207
208 if (layout != ANV_IMAGE_LAYOUT_EXPLICIT_AUX) {
209 assert(usage != 0);
210 aux_usage = anv_layout_to_aux_usage(&device->info, image,
211 aspect, usage, layout);
212 }
213
214 isl_surf_usage_flags_t mocs_usage =
215 (usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT) ?
216 ISL_SURF_USAGE_RENDER_TARGET_BIT : ISL_SURF_USAGE_TEXTURE_BIT;
217
218 const struct anv_surface *surface = &image->planes[plane].surface;
219 *blorp_surf = (struct blorp_surf) {
220 .surf = &surface->isl,
221 .addr = {
222 .buffer = image->planes[plane].address.bo,
223 .offset = image->planes[plane].address.offset + surface->offset,
224 .mocs = anv_mocs(device, image->planes[plane].address.bo, mocs_usage),
225 },
226 };
227
228 if (aux_usage != ISL_AUX_USAGE_NONE) {
229 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface;
230 blorp_surf->aux_surf = &aux_surface->isl,
231 blorp_surf->aux_addr = (struct blorp_address) {
232 .buffer = image->planes[plane].address.bo,
233 .offset = image->planes[plane].address.offset + aux_surface->offset,
234 .mocs = anv_mocs(device, image->planes[plane].address.bo, 0),
235 };
236 blorp_surf->aux_usage = aux_usage;
237
238 /* If we're doing a partial resolve, then we need the indirect clear
239 * color. If we are doing a fast clear and want to store/update the
240 * clear color, we also pass the address to blorp, otherwise it will only
241 * stomp the CCS to a particular value and won't care about format or
242 * clear value
243 */
244 if (aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
245 const struct anv_address clear_color_addr =
246 anv_image_get_clear_color_addr(device, image, aspect);
247 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
248 } else if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT) {
249 if (device->info.gen >= 10) {
250 /* Vulkan always clears to 1.0. On gen < 10, we set that directly
251 * in the state packet. For gen >= 10, must provide the clear
252 * value in a buffer. We have a single global buffer that stores
253 * the 1.0 value.
254 */
255 const struct anv_address clear_color_addr = (struct anv_address) {
256 .bo = device->hiz_clear_bo,
257 };
258 blorp_surf->clear_color_addr =
259 anv_to_blorp_address(clear_color_addr);
260 } else {
261 blorp_surf->clear_color = (union isl_color_value) {
262 .f32 = { ANV_HZ_FC_VAL },
263 };
264 }
265 }
266 }
267 }
268
269 static bool
get_blorp_surf_for_anv_shadow_image(const struct anv_device * device,const struct anv_image * image,VkImageAspectFlags aspect,struct blorp_surf * blorp_surf)270 get_blorp_surf_for_anv_shadow_image(const struct anv_device *device,
271 const struct anv_image *image,
272 VkImageAspectFlags aspect,
273 struct blorp_surf *blorp_surf)
274 {
275
276 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
277 if (image->planes[plane].shadow_surface.isl.size_B == 0)
278 return false;
279
280 *blorp_surf = (struct blorp_surf) {
281 .surf = &image->planes[plane].shadow_surface.isl,
282 .addr = {
283 .buffer = image->planes[plane].address.bo,
284 .offset = image->planes[plane].address.offset +
285 image->planes[plane].shadow_surface.offset,
286 .mocs = anv_mocs(device, image->planes[plane].address.bo,
287 ISL_SURF_USAGE_RENDER_TARGET_BIT),
288 },
289 };
290
291 return true;
292 }
293
294 static void
copy_image(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,struct anv_image * src_image,VkImageLayout src_image_layout,struct anv_image * dst_image,VkImageLayout dst_image_layout,const VkImageCopy2KHR * region)295 copy_image(struct anv_cmd_buffer *cmd_buffer,
296 struct blorp_batch *batch,
297 struct anv_image *src_image,
298 VkImageLayout src_image_layout,
299 struct anv_image *dst_image,
300 VkImageLayout dst_image_layout,
301 const VkImageCopy2KHR *region)
302 {
303 VkOffset3D srcOffset =
304 anv_sanitize_image_offset(src_image->type, region->srcOffset);
305 VkOffset3D dstOffset =
306 anv_sanitize_image_offset(dst_image->type, region->dstOffset);
307 VkExtent3D extent =
308 anv_sanitize_image_extent(src_image->type, region->extent);
309
310 const uint32_t dst_level = region->dstSubresource.mipLevel;
311 unsigned dst_base_layer, layer_count;
312 if (dst_image->type == VK_IMAGE_TYPE_3D) {
313 dst_base_layer = region->dstOffset.z;
314 layer_count = region->extent.depth;
315 } else {
316 dst_base_layer = region->dstSubresource.baseArrayLayer;
317 layer_count =
318 anv_get_layerCount(dst_image, ®ion->dstSubresource);
319 }
320
321 const uint32_t src_level = region->srcSubresource.mipLevel;
322 unsigned src_base_layer;
323 if (src_image->type == VK_IMAGE_TYPE_3D) {
324 src_base_layer = region->srcOffset.z;
325 } else {
326 src_base_layer = region->srcSubresource.baseArrayLayer;
327 assert(layer_count ==
328 anv_get_layerCount(src_image, ®ion->srcSubresource));
329 }
330
331 VkImageAspectFlags src_mask = region->srcSubresource.aspectMask,
332 dst_mask = region->dstSubresource.aspectMask;
333
334 assert(anv_image_aspects_compatible(src_mask, dst_mask));
335
336 if (util_bitcount(src_mask) > 1) {
337 uint32_t aspect_bit;
338 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) {
339 struct blorp_surf src_surf, dst_surf;
340 get_blorp_surf_for_anv_image(cmd_buffer->device,
341 src_image, 1UL << aspect_bit,
342 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
343 src_image_layout, ISL_AUX_USAGE_NONE,
344 &src_surf);
345 get_blorp_surf_for_anv_image(cmd_buffer->device,
346 dst_image, 1UL << aspect_bit,
347 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
348 dst_image_layout, ISL_AUX_USAGE_NONE,
349 &dst_surf);
350 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
351 1UL << aspect_bit,
352 dst_surf.aux_usage, dst_level,
353 dst_base_layer, layer_count);
354
355 for (unsigned i = 0; i < layer_count; i++) {
356 blorp_copy(batch, &src_surf, src_level, src_base_layer + i,
357 &dst_surf, dst_level, dst_base_layer + i,
358 srcOffset.x, srcOffset.y,
359 dstOffset.x, dstOffset.y,
360 extent.width, extent.height);
361 }
362
363 struct blorp_surf dst_shadow_surf;
364 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
365 dst_image,
366 1UL << aspect_bit,
367 &dst_shadow_surf)) {
368 for (unsigned i = 0; i < layer_count; i++) {
369 blorp_copy(batch, &src_surf, src_level, src_base_layer + i,
370 &dst_shadow_surf, dst_level, dst_base_layer + i,
371 srcOffset.x, srcOffset.y,
372 dstOffset.x, dstOffset.y,
373 extent.width, extent.height);
374 }
375 }
376 }
377 } else {
378 struct blorp_surf src_surf, dst_surf;
379 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, src_mask,
380 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
381 src_image_layout, ISL_AUX_USAGE_NONE,
382 &src_surf);
383 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, dst_mask,
384 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
385 dst_image_layout, ISL_AUX_USAGE_NONE,
386 &dst_surf);
387 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask,
388 dst_surf.aux_usage, dst_level,
389 dst_base_layer, layer_count);
390
391 for (unsigned i = 0; i < layer_count; i++) {
392 blorp_copy(batch, &src_surf, src_level, src_base_layer + i,
393 &dst_surf, dst_level, dst_base_layer + i,
394 srcOffset.x, srcOffset.y,
395 dstOffset.x, dstOffset.y,
396 extent.width, extent.height);
397 }
398
399 struct blorp_surf dst_shadow_surf;
400 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
401 dst_image, dst_mask,
402 &dst_shadow_surf)) {
403 for (unsigned i = 0; i < layer_count; i++) {
404 blorp_copy(batch, &src_surf, src_level, src_base_layer + i,
405 &dst_shadow_surf, dst_level, dst_base_layer + i,
406 srcOffset.x, srcOffset.y,
407 dstOffset.x, dstOffset.y,
408 extent.width, extent.height);
409 }
410 }
411 }
412 }
413
414
anv_CmdCopyImage(VkCommandBuffer commandBuffer,VkImage srcImage,VkImageLayout srcImageLayout,VkImage dstImage,VkImageLayout dstImageLayout,uint32_t regionCount,const VkImageCopy * pRegions)415 void anv_CmdCopyImage(
416 VkCommandBuffer commandBuffer,
417 VkImage srcImage,
418 VkImageLayout srcImageLayout,
419 VkImage dstImage,
420 VkImageLayout dstImageLayout,
421 uint32_t regionCount,
422 const VkImageCopy* pRegions)
423 {
424 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
425 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
426 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
427
428 struct blorp_batch batch;
429 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
430
431 for (unsigned r = 0; r < regionCount; r++) {
432 VkImageCopy2KHR copy = {
433 .sType = VK_STRUCTURE_TYPE_IMAGE_COPY_2_KHR,
434 .srcSubresource = pRegions[r].srcSubresource,
435 .srcOffset = pRegions[r].srcOffset,
436 .dstSubresource = pRegions[r].dstSubresource,
437 .dstOffset = pRegions[r].dstOffset,
438 .extent = pRegions[r].extent,
439 };
440
441 copy_image(cmd_buffer, &batch,
442 src_image, srcImageLayout,
443 dst_image, dstImageLayout,
444 ©);
445 }
446
447 blorp_batch_finish(&batch);
448 }
449
anv_CmdCopyImage2KHR(VkCommandBuffer commandBuffer,const VkCopyImageInfo2KHR * pCopyImageInfo)450 void anv_CmdCopyImage2KHR(
451 VkCommandBuffer commandBuffer,
452 const VkCopyImageInfo2KHR* pCopyImageInfo)
453 {
454 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
455 ANV_FROM_HANDLE(anv_image, src_image, pCopyImageInfo->srcImage);
456 ANV_FROM_HANDLE(anv_image, dst_image, pCopyImageInfo->dstImage);
457
458 struct blorp_batch batch;
459 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
460
461 for (unsigned r = 0; r < pCopyImageInfo->regionCount; r++) {
462 copy_image(cmd_buffer, &batch,
463 src_image, pCopyImageInfo->srcImageLayout,
464 dst_image, pCopyImageInfo->dstImageLayout,
465 &pCopyImageInfo->pRegions[r]);
466 }
467
468 blorp_batch_finish(&batch);
469 }
470
471 static enum isl_format
isl_format_for_size(unsigned size_B)472 isl_format_for_size(unsigned size_B)
473 {
474 /* Prefer 32-bit per component formats for CmdFillBuffer */
475 switch (size_B) {
476 case 1: return ISL_FORMAT_R8_UINT;
477 case 2: return ISL_FORMAT_R16_UINT;
478 case 3: return ISL_FORMAT_R8G8B8_UINT;
479 case 4: return ISL_FORMAT_R32_UINT;
480 case 6: return ISL_FORMAT_R16G16B16_UINT;
481 case 8: return ISL_FORMAT_R32G32_UINT;
482 case 12: return ISL_FORMAT_R32G32B32_UINT;
483 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
484 default:
485 unreachable("Unknown format size");
486 }
487 }
488
489 static void
copy_buffer_to_image(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,struct anv_buffer * anv_buffer,struct anv_image * anv_image,VkImageLayout image_layout,const VkBufferImageCopy2KHR * region,bool buffer_to_image)490 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
491 struct blorp_batch *batch,
492 struct anv_buffer *anv_buffer,
493 struct anv_image *anv_image,
494 VkImageLayout image_layout,
495 const VkBufferImageCopy2KHR* region,
496 bool buffer_to_image)
497 {
498 struct {
499 struct blorp_surf surf;
500 uint32_t level;
501 VkOffset3D offset;
502 } image, buffer, *src, *dst;
503
504 buffer.level = 0;
505 buffer.offset = (VkOffset3D) { 0, 0, 0 };
506
507 if (buffer_to_image) {
508 src = &buffer;
509 dst = ℑ
510 } else {
511 src = ℑ
512 dst = &buffer;
513 }
514
515 const VkImageAspectFlags aspect = region->imageSubresource.aspectMask;
516
517 get_blorp_surf_for_anv_image(cmd_buffer->device, anv_image, aspect,
518 buffer_to_image ?
519 VK_IMAGE_USAGE_TRANSFER_DST_BIT :
520 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
521 image_layout, ISL_AUX_USAGE_NONE,
522 &image.surf);
523 image.offset =
524 anv_sanitize_image_offset(anv_image->type, region->imageOffset);
525 image.level = region->imageSubresource.mipLevel;
526
527 VkExtent3D extent =
528 anv_sanitize_image_extent(anv_image->type, region->imageExtent);
529 if (anv_image->type != VK_IMAGE_TYPE_3D) {
530 image.offset.z = region->imageSubresource.baseArrayLayer;
531 extent.depth =
532 anv_get_layerCount(anv_image, ®ion->imageSubresource);
533 }
534
535 const enum isl_format linear_format =
536 anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format,
537 aspect, VK_IMAGE_TILING_LINEAR);
538 const struct isl_format_layout *linear_fmtl =
539 isl_format_get_layout(linear_format);
540
541 const uint32_t buffer_row_length =
542 region->bufferRowLength ?
543 region->bufferRowLength : extent.width;
544
545 const uint32_t buffer_image_height =
546 region->bufferImageHeight ?
547 region->bufferImageHeight : extent.height;
548
549 const uint32_t buffer_row_pitch =
550 DIV_ROUND_UP(buffer_row_length, linear_fmtl->bw) *
551 (linear_fmtl->bpb / 8);
552
553 const uint32_t buffer_layer_stride =
554 DIV_ROUND_UP(buffer_image_height, linear_fmtl->bh) *
555 buffer_row_pitch;
556
557 /* Some formats have additional restrictions which may cause ISL to
558 * fail to create a surface for us. Some examples include:
559 *
560 * 1. ASTC formats are not allowed to be LINEAR and must be tiled
561 * 2. YCbCr formats have to have 2-pixel aligned strides
562 *
563 * To avoid these issues, we always bind the buffer as if it's a
564 * "normal" format like RGBA32_UINT. Since we're using blorp_copy,
565 * the format doesn't matter as long as it has the right bpb.
566 */
567 const VkExtent2D buffer_extent = {
568 .width = DIV_ROUND_UP(extent.width, linear_fmtl->bw),
569 .height = DIV_ROUND_UP(extent.height, linear_fmtl->bh),
570 };
571 const enum isl_format buffer_format =
572 isl_format_for_size(linear_fmtl->bpb / 8);
573
574 struct isl_surf buffer_isl_surf;
575 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
576 anv_buffer, region->bufferOffset,
577 buffer_extent.width, buffer_extent.height,
578 buffer_row_pitch, buffer_format, false,
579 &buffer.surf, &buffer_isl_surf);
580
581 bool dst_has_shadow = false;
582 struct blorp_surf dst_shadow_surf;
583 if (&image == dst) {
584 /* In this case, the source is the buffer and, since blorp takes its
585 * copy dimensions in terms of the source format, we have to use the
586 * scaled down version for compressed textures because the source
587 * format is an RGB format.
588 */
589 extent.width = buffer_extent.width;
590 extent.height = buffer_extent.height;
591
592 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image,
593 aspect, dst->surf.aux_usage,
594 dst->level,
595 dst->offset.z, extent.depth);
596
597 dst_has_shadow =
598 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
599 anv_image, aspect,
600 &dst_shadow_surf);
601 }
602
603 for (unsigned z = 0; z < extent.depth; z++) {
604 blorp_copy(batch, &src->surf, src->level, src->offset.z,
605 &dst->surf, dst->level, dst->offset.z,
606 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
607 extent.width, extent.height);
608
609 if (dst_has_shadow) {
610 blorp_copy(batch, &src->surf, src->level, src->offset.z,
611 &dst_shadow_surf, dst->level, dst->offset.z,
612 src->offset.x, src->offset.y,
613 dst->offset.x, dst->offset.y,
614 extent.width, extent.height);
615 }
616
617 image.offset.z++;
618 buffer.surf.addr.offset += buffer_layer_stride;
619 }
620 }
621
anv_CmdCopyBufferToImage(VkCommandBuffer commandBuffer,VkBuffer srcBuffer,VkImage dstImage,VkImageLayout dstImageLayout,uint32_t regionCount,const VkBufferImageCopy * pRegions)622 void anv_CmdCopyBufferToImage(
623 VkCommandBuffer commandBuffer,
624 VkBuffer srcBuffer,
625 VkImage dstImage,
626 VkImageLayout dstImageLayout,
627 uint32_t regionCount,
628 const VkBufferImageCopy* pRegions)
629 {
630 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
631 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
632 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
633
634 struct blorp_batch batch;
635 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
636
637 for (unsigned r = 0; r < regionCount; r++) {
638 VkBufferImageCopy2KHR copy = {
639 .sType = VK_STRUCTURE_TYPE_BUFFER_IMAGE_COPY_2_KHR,
640 .bufferOffset = pRegions[r].bufferOffset,
641 .bufferRowLength = pRegions[r].bufferRowLength,
642 .bufferImageHeight = pRegions[r].bufferImageHeight,
643 .imageSubresource = pRegions[r].imageSubresource,
644 .imageOffset = pRegions[r].imageOffset,
645 .imageExtent = pRegions[r].imageExtent,
646 };
647
648 copy_buffer_to_image(cmd_buffer, &batch, src_buffer, dst_image,
649 dstImageLayout, ©, true);
650 }
651
652 blorp_batch_finish(&batch);
653 }
654
anv_CmdCopyBufferToImage2KHR(VkCommandBuffer commandBuffer,const VkCopyBufferToImageInfo2KHR * pCopyBufferToImageInfo)655 void anv_CmdCopyBufferToImage2KHR(
656 VkCommandBuffer commandBuffer,
657 const VkCopyBufferToImageInfo2KHR* pCopyBufferToImageInfo)
658 {
659 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
660 ANV_FROM_HANDLE(anv_buffer, src_buffer, pCopyBufferToImageInfo->srcBuffer);
661 ANV_FROM_HANDLE(anv_image, dst_image, pCopyBufferToImageInfo->dstImage);
662
663 struct blorp_batch batch;
664 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
665
666 for (unsigned r = 0; r < pCopyBufferToImageInfo->regionCount; r++) {
667 copy_buffer_to_image(cmd_buffer, &batch, src_buffer, dst_image,
668 pCopyBufferToImageInfo->dstImageLayout,
669 &pCopyBufferToImageInfo->pRegions[r], true);
670 }
671
672 blorp_batch_finish(&batch);
673 }
674
anv_CmdCopyImageToBuffer(VkCommandBuffer commandBuffer,VkImage srcImage,VkImageLayout srcImageLayout,VkBuffer dstBuffer,uint32_t regionCount,const VkBufferImageCopy * pRegions)675 void anv_CmdCopyImageToBuffer(
676 VkCommandBuffer commandBuffer,
677 VkImage srcImage,
678 VkImageLayout srcImageLayout,
679 VkBuffer dstBuffer,
680 uint32_t regionCount,
681 const VkBufferImageCopy* pRegions)
682 {
683 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
684 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
685 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
686
687 struct blorp_batch batch;
688 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
689
690 for (unsigned r = 0; r < regionCount; r++) {
691 VkBufferImageCopy2KHR copy = {
692 .sType = VK_STRUCTURE_TYPE_BUFFER_IMAGE_COPY_2_KHR,
693 .bufferOffset = pRegions[r].bufferOffset,
694 .bufferRowLength = pRegions[r].bufferRowLength,
695 .bufferImageHeight = pRegions[r].bufferImageHeight,
696 .imageSubresource = pRegions[r].imageSubresource,
697 .imageOffset = pRegions[r].imageOffset,
698 .imageExtent = pRegions[r].imageExtent,
699 };
700
701 copy_buffer_to_image(cmd_buffer, &batch, dst_buffer, src_image,
702 srcImageLayout, ©, false);
703 }
704
705 blorp_batch_finish(&batch);
706
707 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
708 }
709
anv_CmdCopyImageToBuffer2KHR(VkCommandBuffer commandBuffer,const VkCopyImageToBufferInfo2KHR * pCopyImageToBufferInfo)710 void anv_CmdCopyImageToBuffer2KHR(
711 VkCommandBuffer commandBuffer,
712 const VkCopyImageToBufferInfo2KHR* pCopyImageToBufferInfo)
713 {
714 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
715 ANV_FROM_HANDLE(anv_image, src_image, pCopyImageToBufferInfo->srcImage);
716 ANV_FROM_HANDLE(anv_buffer, dst_buffer, pCopyImageToBufferInfo->dstBuffer);
717
718 struct blorp_batch batch;
719 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
720
721 for (unsigned r = 0; r < pCopyImageToBufferInfo->regionCount; r++) {
722 copy_buffer_to_image(cmd_buffer, &batch, dst_buffer, src_image,
723 pCopyImageToBufferInfo->srcImageLayout,
724 &pCopyImageToBufferInfo->pRegions[r], false);
725 }
726
727 blorp_batch_finish(&batch);
728
729 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
730 }
731
732 static bool
flip_coords(unsigned * src0,unsigned * src1,unsigned * dst0,unsigned * dst1)733 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
734 {
735 bool flip = false;
736 if (*src0 > *src1) {
737 unsigned tmp = *src0;
738 *src0 = *src1;
739 *src1 = tmp;
740 flip = !flip;
741 }
742
743 if (*dst0 > *dst1) {
744 unsigned tmp = *dst0;
745 *dst0 = *dst1;
746 *dst1 = tmp;
747 flip = !flip;
748 }
749
750 return flip;
751 }
752
753 static void
blit_image(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,struct anv_image * src_image,VkImageLayout src_image_layout,struct anv_image * dst_image,VkImageLayout dst_image_layout,const VkImageBlit2KHR * region,VkFilter filter)754 blit_image(struct anv_cmd_buffer *cmd_buffer,
755 struct blorp_batch *batch,
756 struct anv_image *src_image,
757 VkImageLayout src_image_layout,
758 struct anv_image *dst_image,
759 VkImageLayout dst_image_layout,
760 const VkImageBlit2KHR *region,
761 VkFilter filter)
762 {
763 const VkImageSubresourceLayers *src_res = ®ion->srcSubresource;
764 const VkImageSubresourceLayers *dst_res = ®ion->dstSubresource;
765
766 struct blorp_surf src, dst;
767
768 enum blorp_filter blorp_filter;
769 switch (filter) {
770 case VK_FILTER_NEAREST:
771 blorp_filter = BLORP_FILTER_NEAREST;
772 break;
773 case VK_FILTER_LINEAR:
774 blorp_filter = BLORP_FILTER_BILINEAR;
775 break;
776 default:
777 unreachable("Invalid filter");
778 }
779
780 assert(anv_image_aspects_compatible(src_res->aspectMask,
781 dst_res->aspectMask));
782
783 uint32_t aspect_bit;
784 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_res->aspectMask) {
785 get_blorp_surf_for_anv_image(cmd_buffer->device,
786 src_image, 1U << aspect_bit,
787 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
788 src_image_layout, ISL_AUX_USAGE_NONE, &src);
789 get_blorp_surf_for_anv_image(cmd_buffer->device,
790 dst_image, 1U << aspect_bit,
791 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
792 dst_image_layout, ISL_AUX_USAGE_NONE, &dst);
793
794 struct anv_format_plane src_format =
795 anv_get_format_plane(&cmd_buffer->device->info, src_image->vk_format,
796 1U << aspect_bit, src_image->tiling);
797 struct anv_format_plane dst_format =
798 anv_get_format_plane(&cmd_buffer->device->info, dst_image->vk_format,
799 1U << aspect_bit, dst_image->tiling);
800
801 unsigned dst_start, dst_end;
802 if (dst_image->type == VK_IMAGE_TYPE_3D) {
803 assert(dst_res->baseArrayLayer == 0);
804 dst_start = region->dstOffsets[0].z;
805 dst_end = region->dstOffsets[1].z;
806 } else {
807 dst_start = dst_res->baseArrayLayer;
808 dst_end = dst_start + anv_get_layerCount(dst_image, dst_res);
809 }
810
811 unsigned src_start, src_end;
812 if (src_image->type == VK_IMAGE_TYPE_3D) {
813 assert(src_res->baseArrayLayer == 0);
814 src_start = region->srcOffsets[0].z;
815 src_end = region->srcOffsets[1].z;
816 } else {
817 src_start = src_res->baseArrayLayer;
818 src_end = src_start + anv_get_layerCount(src_image, src_res);
819 }
820
821 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
822 const unsigned num_layers = dst_end - dst_start;
823 float src_z_step = (float)(src_end - src_start) / (float)num_layers;
824
825 /* There is no interpolation to the pixel center during rendering, so
826 * add the 0.5 offset ourselves here. */
827 float depth_center_offset = 0;
828 if (src_image->type == VK_IMAGE_TYPE_3D)
829 depth_center_offset = 0.5 / num_layers * (src_end - src_start);
830
831 if (flip_z) {
832 src_start = src_end;
833 src_z_step *= -1;
834 depth_center_offset *= -1;
835 }
836
837 unsigned src_x0 = region->srcOffsets[0].x;
838 unsigned src_x1 = region->srcOffsets[1].x;
839 unsigned dst_x0 = region->dstOffsets[0].x;
840 unsigned dst_x1 = region->dstOffsets[1].x;
841 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
842
843 unsigned src_y0 = region->srcOffsets[0].y;
844 unsigned src_y1 = region->srcOffsets[1].y;
845 unsigned dst_y0 = region->dstOffsets[0].y;
846 unsigned dst_y1 = region->dstOffsets[1].y;
847 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
848
849 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
850 1U << aspect_bit,
851 dst.aux_usage,
852 dst_res->mipLevel,
853 dst_start, num_layers);
854
855 for (unsigned i = 0; i < num_layers; i++) {
856 unsigned dst_z = dst_start + i;
857 float src_z = src_start + i * src_z_step + depth_center_offset;
858
859 blorp_blit(batch, &src, src_res->mipLevel, src_z,
860 src_format.isl_format, src_format.swizzle,
861 &dst, dst_res->mipLevel, dst_z,
862 dst_format.isl_format, dst_format.swizzle,
863 src_x0, src_y0, src_x1, src_y1,
864 dst_x0, dst_y0, dst_x1, dst_y1,
865 blorp_filter, flip_x, flip_y);
866 }
867 }
868 }
869
anv_CmdBlitImage(VkCommandBuffer commandBuffer,VkImage srcImage,VkImageLayout srcImageLayout,VkImage dstImage,VkImageLayout dstImageLayout,uint32_t regionCount,const VkImageBlit * pRegions,VkFilter filter)870 void anv_CmdBlitImage(
871 VkCommandBuffer commandBuffer,
872 VkImage srcImage,
873 VkImageLayout srcImageLayout,
874 VkImage dstImage,
875 VkImageLayout dstImageLayout,
876 uint32_t regionCount,
877 const VkImageBlit* pRegions,
878 VkFilter filter)
879 {
880 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
881 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
882 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
883
884 struct blorp_batch batch;
885 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
886
887 for (unsigned r = 0; r < regionCount; r++) {
888 VkImageBlit2KHR blit = {
889 .sType = VK_STRUCTURE_TYPE_IMAGE_BLIT_2_KHR,
890 .srcSubresource = pRegions[r].srcSubresource,
891 .srcOffsets = {
892 pRegions[r].srcOffsets[0],
893 pRegions[r].srcOffsets[1],
894 },
895 .dstSubresource = pRegions[r].dstSubresource,
896 .dstOffsets = {
897 pRegions[r].dstOffsets[0],
898 pRegions[r].dstOffsets[1],
899 },
900 };
901
902 blit_image(cmd_buffer, &batch,
903 src_image, srcImageLayout,
904 dst_image, dstImageLayout,
905 &blit, filter);
906 }
907
908 blorp_batch_finish(&batch);
909 }
910
anv_CmdBlitImage2KHR(VkCommandBuffer commandBuffer,const VkBlitImageInfo2KHR * pBlitImageInfo)911 void anv_CmdBlitImage2KHR(
912 VkCommandBuffer commandBuffer,
913 const VkBlitImageInfo2KHR* pBlitImageInfo)
914 {
915 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
916 ANV_FROM_HANDLE(anv_image, src_image, pBlitImageInfo->srcImage);
917 ANV_FROM_HANDLE(anv_image, dst_image, pBlitImageInfo->dstImage);
918
919 struct blorp_batch batch;
920 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
921
922 for (unsigned r = 0; r < pBlitImageInfo->regionCount; r++) {
923 blit_image(cmd_buffer, &batch,
924 src_image, pBlitImageInfo->srcImageLayout,
925 dst_image, pBlitImageInfo->dstImageLayout,
926 &pBlitImageInfo->pRegions[r], pBlitImageInfo->filter);
927 }
928
929 blorp_batch_finish(&batch);
930 }
931
932 /**
933 * Returns the greatest common divisor of a and b that is a power of two.
934 */
935 static uint64_t
gcd_pow2_u64(uint64_t a,uint64_t b)936 gcd_pow2_u64(uint64_t a, uint64_t b)
937 {
938 assert(a > 0 || b > 0);
939
940 unsigned a_log2 = ffsll(a) - 1;
941 unsigned b_log2 = ffsll(b) - 1;
942
943 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
944 * case, the MIN2() will take the other one. If both are 0 then we will
945 * hit the assert above.
946 */
947 return 1 << MIN2(a_log2, b_log2);
948 }
949
950 /* This is maximum possible width/height our HW can handle */
951 #define MAX_SURFACE_DIM (1ull << 14)
952
953 static void
copy_buffer(struct anv_device * device,struct blorp_batch * batch,struct anv_buffer * src_buffer,struct anv_buffer * dst_buffer,const VkBufferCopy2KHR * region)954 copy_buffer(struct anv_device *device,
955 struct blorp_batch *batch,
956 struct anv_buffer *src_buffer,
957 struct anv_buffer *dst_buffer,
958 const VkBufferCopy2KHR *region)
959 {
960 struct blorp_address src = {
961 .buffer = src_buffer->address.bo,
962 .offset = src_buffer->address.offset + region->srcOffset,
963 .mocs = anv_mocs(device, src_buffer->address.bo,
964 ISL_SURF_USAGE_TEXTURE_BIT),
965 };
966 struct blorp_address dst = {
967 .buffer = dst_buffer->address.bo,
968 .offset = dst_buffer->address.offset + region->dstOffset,
969 .mocs = anv_mocs(device, dst_buffer->address.bo,
970 ISL_SURF_USAGE_RENDER_TARGET_BIT),
971 };
972
973 blorp_buffer_copy(batch, src, dst, region->size);
974 }
975
anv_CmdCopyBuffer(VkCommandBuffer commandBuffer,VkBuffer srcBuffer,VkBuffer dstBuffer,uint32_t regionCount,const VkBufferCopy * pRegions)976 void anv_CmdCopyBuffer(
977 VkCommandBuffer commandBuffer,
978 VkBuffer srcBuffer,
979 VkBuffer dstBuffer,
980 uint32_t regionCount,
981 const VkBufferCopy* pRegions)
982 {
983 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
984 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
985 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
986
987 struct blorp_batch batch;
988 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
989
990 for (unsigned r = 0; r < regionCount; r++) {
991 VkBufferCopy2KHR copy = {
992 .sType = VK_STRUCTURE_TYPE_BUFFER_COPY_2_KHR,
993 .srcOffset = pRegions[r].srcOffset,
994 .dstOffset = pRegions[r].dstOffset,
995 .size = pRegions[r].size,
996 };
997
998 copy_buffer(cmd_buffer->device, &batch, src_buffer, dst_buffer, ©);
999 }
1000
1001 blorp_batch_finish(&batch);
1002
1003 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
1004 }
1005
anv_CmdCopyBuffer2KHR(VkCommandBuffer commandBuffer,const VkCopyBufferInfo2KHR * pCopyBufferInfo)1006 void anv_CmdCopyBuffer2KHR(
1007 VkCommandBuffer commandBuffer,
1008 const VkCopyBufferInfo2KHR* pCopyBufferInfo)
1009 {
1010 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1011 ANV_FROM_HANDLE(anv_buffer, src_buffer, pCopyBufferInfo->srcBuffer);
1012 ANV_FROM_HANDLE(anv_buffer, dst_buffer, pCopyBufferInfo->dstBuffer);
1013
1014 struct blorp_batch batch;
1015 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1016
1017 for (unsigned r = 0; r < pCopyBufferInfo->regionCount; r++) {
1018 copy_buffer(cmd_buffer->device, &batch, src_buffer, dst_buffer,
1019 &pCopyBufferInfo->pRegions[r]);
1020 }
1021
1022 blorp_batch_finish(&batch);
1023
1024 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
1025 }
1026
1027
anv_CmdUpdateBuffer(VkCommandBuffer commandBuffer,VkBuffer dstBuffer,VkDeviceSize dstOffset,VkDeviceSize dataSize,const void * pData)1028 void anv_CmdUpdateBuffer(
1029 VkCommandBuffer commandBuffer,
1030 VkBuffer dstBuffer,
1031 VkDeviceSize dstOffset,
1032 VkDeviceSize dataSize,
1033 const void* pData)
1034 {
1035 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1036 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
1037
1038 struct blorp_batch batch;
1039 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1040
1041 /* We can't quite grab a full block because the state stream needs a
1042 * little data at the top to build its linked list.
1043 */
1044 const uint32_t max_update_size =
1045 cmd_buffer->device->dynamic_state_pool.block_size - 64;
1046
1047 assert(max_update_size < MAX_SURFACE_DIM * 4);
1048
1049 /* We're about to read data that was written from the CPU. Flush the
1050 * texture cache so we don't get anything stale.
1051 */
1052 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
1053
1054 while (dataSize) {
1055 const uint32_t copy_size = MIN2(dataSize, max_update_size);
1056
1057 struct anv_state tmp_data =
1058 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
1059
1060 memcpy(tmp_data.map, pData, copy_size);
1061
1062 struct blorp_address src = {
1063 .buffer = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
1064 .offset = tmp_data.offset,
1065 .mocs = isl_mocs(&cmd_buffer->device->isl_dev,
1066 ISL_SURF_USAGE_TEXTURE_BIT)
1067 };
1068 struct blorp_address dst = {
1069 .buffer = dst_buffer->address.bo,
1070 .offset = dst_buffer->address.offset + dstOffset,
1071 .mocs = anv_mocs(cmd_buffer->device, dst_buffer->address.bo,
1072 ISL_SURF_USAGE_RENDER_TARGET_BIT),
1073 };
1074
1075 blorp_buffer_copy(&batch, src, dst, copy_size);
1076
1077 dataSize -= copy_size;
1078 dstOffset += copy_size;
1079 pData = (void *)pData + copy_size;
1080 }
1081
1082 blorp_batch_finish(&batch);
1083
1084 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
1085 }
1086
anv_CmdFillBuffer(VkCommandBuffer commandBuffer,VkBuffer dstBuffer,VkDeviceSize dstOffset,VkDeviceSize fillSize,uint32_t data)1087 void anv_CmdFillBuffer(
1088 VkCommandBuffer commandBuffer,
1089 VkBuffer dstBuffer,
1090 VkDeviceSize dstOffset,
1091 VkDeviceSize fillSize,
1092 uint32_t data)
1093 {
1094 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1095 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
1096 struct blorp_surf surf;
1097 struct isl_surf isl_surf;
1098
1099 struct blorp_batch batch;
1100 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1101
1102 fillSize = anv_buffer_get_range(dst_buffer, dstOffset, fillSize);
1103
1104 /* From the Vulkan spec:
1105 *
1106 * "size is the number of bytes to fill, and must be either a multiple
1107 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of
1108 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the
1109 * buffer is not a multiple of 4, then the nearest smaller multiple is
1110 * used."
1111 */
1112 fillSize &= ~3ull;
1113
1114 /* First, we compute the biggest format that can be used with the
1115 * given offsets and size.
1116 */
1117 int bs = 16;
1118 bs = gcd_pow2_u64(bs, dstOffset);
1119 bs = gcd_pow2_u64(bs, fillSize);
1120 enum isl_format isl_format = isl_format_for_size(bs);
1121
1122 union isl_color_value color = {
1123 .u32 = { data, data, data, data },
1124 };
1125
1126 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs;
1127 while (fillSize >= max_fill_size) {
1128 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
1129 dst_buffer, dstOffset,
1130 MAX_SURFACE_DIM, MAX_SURFACE_DIM,
1131 MAX_SURFACE_DIM * bs, isl_format, true,
1132 &surf, &isl_surf);
1133
1134 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
1135 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM,
1136 color, NULL);
1137 fillSize -= max_fill_size;
1138 dstOffset += max_fill_size;
1139 }
1140
1141 uint64_t height = fillSize / (MAX_SURFACE_DIM * bs);
1142 assert(height < MAX_SURFACE_DIM);
1143 if (height != 0) {
1144 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs;
1145 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
1146 dst_buffer, dstOffset,
1147 MAX_SURFACE_DIM, height,
1148 MAX_SURFACE_DIM * bs, isl_format, true,
1149 &surf, &isl_surf);
1150
1151 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
1152 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height,
1153 color, NULL);
1154 fillSize -= rect_fill_size;
1155 dstOffset += rect_fill_size;
1156 }
1157
1158 if (fillSize != 0) {
1159 const uint32_t width = fillSize / bs;
1160 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
1161 dst_buffer, dstOffset,
1162 width, 1,
1163 width * bs, isl_format, true,
1164 &surf, &isl_surf);
1165
1166 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
1167 0, 0, 1, 0, 0, width, 1,
1168 color, NULL);
1169 }
1170
1171 blorp_batch_finish(&batch);
1172
1173 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
1174 }
1175
anv_CmdClearColorImage(VkCommandBuffer commandBuffer,VkImage _image,VkImageLayout imageLayout,const VkClearColorValue * pColor,uint32_t rangeCount,const VkImageSubresourceRange * pRanges)1176 void anv_CmdClearColorImage(
1177 VkCommandBuffer commandBuffer,
1178 VkImage _image,
1179 VkImageLayout imageLayout,
1180 const VkClearColorValue* pColor,
1181 uint32_t rangeCount,
1182 const VkImageSubresourceRange* pRanges)
1183 {
1184 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1185 ANV_FROM_HANDLE(anv_image, image, _image);
1186
1187 static const bool color_write_disable[4] = { false, false, false, false };
1188
1189 struct blorp_batch batch;
1190 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1191
1192
1193 for (unsigned r = 0; r < rangeCount; r++) {
1194 if (pRanges[r].aspectMask == 0)
1195 continue;
1196
1197 assert(pRanges[r].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
1198
1199 struct blorp_surf surf;
1200 get_blorp_surf_for_anv_image(cmd_buffer->device,
1201 image, pRanges[r].aspectMask,
1202 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1203 imageLayout, ISL_AUX_USAGE_NONE, &surf);
1204
1205 struct anv_format_plane src_format =
1206 anv_get_format_plane(&cmd_buffer->device->info, image->vk_format,
1207 VK_IMAGE_ASPECT_COLOR_BIT, image->tiling);
1208
1209 unsigned base_layer = pRanges[r].baseArrayLayer;
1210 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
1211
1212 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
1213 const unsigned level = pRanges[r].baseMipLevel + i;
1214 const unsigned level_width = anv_minify(image->extent.width, level);
1215 const unsigned level_height = anv_minify(image->extent.height, level);
1216
1217 if (image->type == VK_IMAGE_TYPE_3D) {
1218 base_layer = 0;
1219 layer_count = anv_minify(image->extent.depth, level);
1220 }
1221
1222 anv_cmd_buffer_mark_image_written(cmd_buffer, image,
1223 pRanges[r].aspectMask,
1224 surf.aux_usage, level,
1225 base_layer, layer_count);
1226
1227 blorp_clear(&batch, &surf,
1228 src_format.isl_format, src_format.swizzle,
1229 level, base_layer, layer_count,
1230 0, 0, level_width, level_height,
1231 vk_to_isl_color(*pColor), color_write_disable);
1232 }
1233 }
1234
1235 blorp_batch_finish(&batch);
1236 }
1237
anv_CmdClearDepthStencilImage(VkCommandBuffer commandBuffer,VkImage image_h,VkImageLayout imageLayout,const VkClearDepthStencilValue * pDepthStencil,uint32_t rangeCount,const VkImageSubresourceRange * pRanges)1238 void anv_CmdClearDepthStencilImage(
1239 VkCommandBuffer commandBuffer,
1240 VkImage image_h,
1241 VkImageLayout imageLayout,
1242 const VkClearDepthStencilValue* pDepthStencil,
1243 uint32_t rangeCount,
1244 const VkImageSubresourceRange* pRanges)
1245 {
1246 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1247 ANV_FROM_HANDLE(anv_image, image, image_h);
1248
1249 struct blorp_batch batch;
1250 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1251
1252 struct blorp_surf depth, stencil, stencil_shadow;
1253 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1254 get_blorp_surf_for_anv_image(cmd_buffer->device,
1255 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1256 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1257 imageLayout, ISL_AUX_USAGE_NONE, &depth);
1258 } else {
1259 memset(&depth, 0, sizeof(depth));
1260 }
1261
1262 bool has_stencil_shadow = false;
1263 if (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1264 get_blorp_surf_for_anv_image(cmd_buffer->device,
1265 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1266 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1267 imageLayout, ISL_AUX_USAGE_NONE, &stencil);
1268
1269 has_stencil_shadow =
1270 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
1271 VK_IMAGE_ASPECT_STENCIL_BIT,
1272 &stencil_shadow);
1273 } else {
1274 memset(&stencil, 0, sizeof(stencil));
1275 }
1276
1277 for (unsigned r = 0; r < rangeCount; r++) {
1278 if (pRanges[r].aspectMask == 0)
1279 continue;
1280
1281 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1282 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1283
1284 unsigned base_layer = pRanges[r].baseArrayLayer;
1285 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
1286
1287 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
1288 const unsigned level = pRanges[r].baseMipLevel + i;
1289 const unsigned level_width = anv_minify(image->extent.width, level);
1290 const unsigned level_height = anv_minify(image->extent.height, level);
1291
1292 if (image->type == VK_IMAGE_TYPE_3D)
1293 layer_count = anv_minify(image->extent.depth, level);
1294
1295 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1296 level, base_layer, layer_count,
1297 0, 0, level_width, level_height,
1298 clear_depth, pDepthStencil->depth,
1299 clear_stencil ? 0xff : 0,
1300 pDepthStencil->stencil);
1301
1302 if (clear_stencil && has_stencil_shadow) {
1303 union isl_color_value stencil_color = {
1304 .u32 = { pDepthStencil->stencil, },
1305 };
1306 blorp_clear(&batch, &stencil_shadow,
1307 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1308 level, base_layer, layer_count,
1309 0, 0, level_width, level_height,
1310 stencil_color, NULL);
1311 }
1312 }
1313 }
1314
1315 blorp_batch_finish(&batch);
1316 }
1317
1318 VkResult
anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer * cmd_buffer,uint32_t num_entries,uint32_t * state_offset,struct anv_state * bt_state)1319 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
1320 uint32_t num_entries,
1321 uint32_t *state_offset,
1322 struct anv_state *bt_state)
1323 {
1324 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1325 state_offset);
1326 if (bt_state->map == NULL) {
1327 /* We ran out of space. Grab a new binding table block. */
1328 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
1329 if (result != VK_SUCCESS)
1330 return result;
1331
1332 /* Re-emit state base addresses so we get the new surface state base
1333 * address before we start emitting binding tables etc.
1334 */
1335 anv_cmd_buffer_emit_state_base_address(cmd_buffer);
1336
1337 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1338 state_offset);
1339 assert(bt_state->map != NULL);
1340 }
1341
1342 return VK_SUCCESS;
1343 }
1344
1345 static VkResult
binding_table_for_surface_state(struct anv_cmd_buffer * cmd_buffer,struct anv_state surface_state,uint32_t * bt_offset)1346 binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer,
1347 struct anv_state surface_state,
1348 uint32_t *bt_offset)
1349 {
1350 uint32_t state_offset;
1351 struct anv_state bt_state;
1352
1353 VkResult result =
1354 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset,
1355 &bt_state);
1356 if (result != VK_SUCCESS)
1357 return result;
1358
1359 uint32_t *bt_map = bt_state.map;
1360 bt_map[0] = surface_state.offset + state_offset;
1361
1362 *bt_offset = bt_state.offset;
1363 return VK_SUCCESS;
1364 }
1365
1366 static void
clear_color_attachment(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,const VkClearAttachment * attachment,uint32_t rectCount,const VkClearRect * pRects)1367 clear_color_attachment(struct anv_cmd_buffer *cmd_buffer,
1368 struct blorp_batch *batch,
1369 const VkClearAttachment *attachment,
1370 uint32_t rectCount, const VkClearRect *pRects)
1371 {
1372 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1373 const uint32_t color_att = attachment->colorAttachment;
1374 assert(color_att < subpass->color_count);
1375 const uint32_t att_idx = subpass->color_attachments[color_att].attachment;
1376
1377 if (att_idx == VK_ATTACHMENT_UNUSED)
1378 return;
1379
1380 struct anv_render_pass_attachment *pass_att =
1381 &cmd_buffer->state.pass->attachments[att_idx];
1382 struct anv_attachment_state *att_state =
1383 &cmd_buffer->state.attachments[att_idx];
1384
1385 uint32_t binding_table;
1386 VkResult result =
1387 binding_table_for_surface_state(cmd_buffer, att_state->color.state,
1388 &binding_table);
1389 if (result != VK_SUCCESS)
1390 return;
1391
1392 union isl_color_value clear_color =
1393 vk_to_isl_color(attachment->clearValue.color);
1394
1395 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1396 if (subpass->view_mask) {
1397 uint32_t view_idx;
1398 for_each_bit(view_idx, subpass->view_mask) {
1399 for (uint32_t r = 0; r < rectCount; ++r) {
1400 const VkOffset2D offset = pRects[r].rect.offset;
1401 const VkExtent2D extent = pRects[r].rect.extent;
1402 blorp_clear_attachments(batch, binding_table,
1403 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1404 view_idx, 1,
1405 offset.x, offset.y,
1406 offset.x + extent.width,
1407 offset.y + extent.height,
1408 true, clear_color, false, 0.0f, 0, 0);
1409 }
1410 }
1411 return;
1412 }
1413
1414 for (uint32_t r = 0; r < rectCount; ++r) {
1415 const VkOffset2D offset = pRects[r].rect.offset;
1416 const VkExtent2D extent = pRects[r].rect.extent;
1417 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1418 blorp_clear_attachments(batch, binding_table,
1419 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1420 pRects[r].baseArrayLayer,
1421 pRects[r].layerCount,
1422 offset.x, offset.y,
1423 offset.x + extent.width, offset.y + extent.height,
1424 true, clear_color, false, 0.0f, 0, 0);
1425 }
1426 }
1427
1428 static void
clear_depth_stencil_attachment(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,const VkClearAttachment * attachment,uint32_t rectCount,const VkClearRect * pRects)1429 clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer,
1430 struct blorp_batch *batch,
1431 const VkClearAttachment *attachment,
1432 uint32_t rectCount, const VkClearRect *pRects)
1433 {
1434 static const union isl_color_value color_value = { .u32 = { 0, } };
1435 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1436 if (!subpass->depth_stencil_attachment)
1437 return;
1438
1439 const uint32_t att_idx = subpass->depth_stencil_attachment->attachment;
1440 assert(att_idx != VK_ATTACHMENT_UNUSED);
1441 struct anv_render_pass_attachment *pass_att =
1442 &cmd_buffer->state.pass->attachments[att_idx];
1443
1444 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1445 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1446
1447 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED;
1448 if (clear_depth) {
1449 depth_format = anv_get_isl_format(&cmd_buffer->device->info,
1450 pass_att->format,
1451 VK_IMAGE_ASPECT_DEPTH_BIT,
1452 VK_IMAGE_TILING_OPTIMAL);
1453 }
1454
1455 uint32_t binding_table;
1456 VkResult result =
1457 binding_table_for_surface_state(cmd_buffer,
1458 cmd_buffer->state.null_surface_state,
1459 &binding_table);
1460 if (result != VK_SUCCESS)
1461 return;
1462
1463 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1464 if (subpass->view_mask) {
1465 uint32_t view_idx;
1466 for_each_bit(view_idx, subpass->view_mask) {
1467 for (uint32_t r = 0; r < rectCount; ++r) {
1468 const VkOffset2D offset = pRects[r].rect.offset;
1469 const VkExtent2D extent = pRects[r].rect.extent;
1470 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1471 blorp_clear_attachments(batch, binding_table,
1472 depth_format, pass_att->samples,
1473 view_idx, 1,
1474 offset.x, offset.y,
1475 offset.x + extent.width,
1476 offset.y + extent.height,
1477 false, color_value,
1478 clear_depth, value.depth,
1479 clear_stencil ? 0xff : 0, value.stencil);
1480 }
1481 }
1482 return;
1483 }
1484
1485 for (uint32_t r = 0; r < rectCount; ++r) {
1486 const VkOffset2D offset = pRects[r].rect.offset;
1487 const VkExtent2D extent = pRects[r].rect.extent;
1488 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1489 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1490 blorp_clear_attachments(batch, binding_table,
1491 depth_format, pass_att->samples,
1492 pRects[r].baseArrayLayer,
1493 pRects[r].layerCount,
1494 offset.x, offset.y,
1495 offset.x + extent.width, offset.y + extent.height,
1496 false, color_value,
1497 clear_depth, value.depth,
1498 clear_stencil ? 0xff : 0, value.stencil);
1499 }
1500 }
1501
anv_CmdClearAttachments(VkCommandBuffer commandBuffer,uint32_t attachmentCount,const VkClearAttachment * pAttachments,uint32_t rectCount,const VkClearRect * pRects)1502 void anv_CmdClearAttachments(
1503 VkCommandBuffer commandBuffer,
1504 uint32_t attachmentCount,
1505 const VkClearAttachment* pAttachments,
1506 uint32_t rectCount,
1507 const VkClearRect* pRects)
1508 {
1509 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1510
1511 /* Because this gets called within a render pass, we tell blorp not to
1512 * trash our depth and stencil buffers.
1513 */
1514 struct blorp_batch batch;
1515 enum blorp_batch_flags flags = BLORP_BATCH_NO_EMIT_DEPTH_STENCIL;
1516 if (cmd_buffer->state.conditional_render_enabled) {
1517 anv_cmd_emit_conditional_render_predicate(cmd_buffer);
1518 flags |= BLORP_BATCH_PREDICATE_ENABLE;
1519 }
1520 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, flags);
1521
1522 for (uint32_t a = 0; a < attachmentCount; ++a) {
1523 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1524 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1525 clear_color_attachment(cmd_buffer, &batch,
1526 &pAttachments[a],
1527 rectCount, pRects);
1528 } else {
1529 clear_depth_stencil_attachment(cmd_buffer, &batch,
1530 &pAttachments[a],
1531 rectCount, pRects);
1532 }
1533 }
1534
1535 blorp_batch_finish(&batch);
1536 }
1537
1538 enum subpass_stage {
1539 SUBPASS_STAGE_LOAD,
1540 SUBPASS_STAGE_DRAW,
1541 SUBPASS_STAGE_RESOLVE,
1542 };
1543
1544 void
anv_image_msaa_resolve(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * src_image,enum isl_aux_usage src_aux_usage,uint32_t src_level,uint32_t src_base_layer,const struct anv_image * dst_image,enum isl_aux_usage dst_aux_usage,uint32_t dst_level,uint32_t dst_base_layer,VkImageAspectFlagBits aspect,uint32_t src_x,uint32_t src_y,uint32_t dst_x,uint32_t dst_y,uint32_t width,uint32_t height,uint32_t layer_count,enum blorp_filter filter)1545 anv_image_msaa_resolve(struct anv_cmd_buffer *cmd_buffer,
1546 const struct anv_image *src_image,
1547 enum isl_aux_usage src_aux_usage,
1548 uint32_t src_level, uint32_t src_base_layer,
1549 const struct anv_image *dst_image,
1550 enum isl_aux_usage dst_aux_usage,
1551 uint32_t dst_level, uint32_t dst_base_layer,
1552 VkImageAspectFlagBits aspect,
1553 uint32_t src_x, uint32_t src_y,
1554 uint32_t dst_x, uint32_t dst_y,
1555 uint32_t width, uint32_t height,
1556 uint32_t layer_count,
1557 enum blorp_filter filter)
1558 {
1559 struct blorp_batch batch;
1560 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1561
1562 assert(src_image->type == VK_IMAGE_TYPE_2D);
1563 assert(src_image->samples > 1);
1564 assert(dst_image->type == VK_IMAGE_TYPE_2D);
1565 assert(dst_image->samples == 1);
1566 assert(src_image->n_planes == dst_image->n_planes);
1567 assert(!src_image->format->can_ycbcr);
1568 assert(!dst_image->format->can_ycbcr);
1569
1570 struct blorp_surf src_surf, dst_surf;
1571 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, aspect,
1572 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
1573 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1574 src_aux_usage, &src_surf);
1575 if (src_aux_usage == ISL_AUX_USAGE_MCS) {
1576 src_surf.clear_color_addr = anv_to_blorp_address(
1577 anv_image_get_clear_color_addr(cmd_buffer->device, src_image,
1578 VK_IMAGE_ASPECT_COLOR_BIT));
1579 }
1580 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, aspect,
1581 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1582 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1583 dst_aux_usage, &dst_surf);
1584 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
1585 aspect, dst_aux_usage,
1586 dst_level, dst_base_layer, layer_count);
1587
1588 if (filter == BLORP_FILTER_NONE) {
1589 /* If no explicit filter is provided, then it's implied by the type of
1590 * the source image.
1591 */
1592 if ((src_surf.surf->usage & ISL_SURF_USAGE_DEPTH_BIT) ||
1593 (src_surf.surf->usage & ISL_SURF_USAGE_STENCIL_BIT) ||
1594 isl_format_has_int_channel(src_surf.surf->format)) {
1595 filter = BLORP_FILTER_SAMPLE_0;
1596 } else {
1597 filter = BLORP_FILTER_AVERAGE;
1598 }
1599 }
1600
1601 for (uint32_t l = 0; l < layer_count; l++) {
1602 blorp_blit(&batch,
1603 &src_surf, src_level, src_base_layer + l,
1604 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1605 &dst_surf, dst_level, dst_base_layer + l,
1606 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1607 src_x, src_y, src_x + width, src_y + height,
1608 dst_x, dst_y, dst_x + width, dst_y + height,
1609 filter, false, false);
1610 }
1611
1612 blorp_batch_finish(&batch);
1613 }
1614
1615 static void
resolve_image(struct anv_cmd_buffer * cmd_buffer,struct anv_image * src_image,VkImageLayout src_image_layout,struct anv_image * dst_image,VkImageLayout dst_image_layout,const VkImageResolve2KHR * region)1616 resolve_image(struct anv_cmd_buffer *cmd_buffer,
1617 struct anv_image *src_image,
1618 VkImageLayout src_image_layout,
1619 struct anv_image *dst_image,
1620 VkImageLayout dst_image_layout,
1621 const VkImageResolve2KHR *region)
1622 {
1623 assert(region->srcSubresource.aspectMask == region->dstSubresource.aspectMask);
1624 assert(anv_get_layerCount(src_image, ®ion->srcSubresource) ==
1625 anv_get_layerCount(dst_image, ®ion->dstSubresource));
1626
1627 const uint32_t layer_count =
1628 anv_get_layerCount(dst_image, ®ion->dstSubresource);
1629
1630 uint32_t aspect_bit;
1631 anv_foreach_image_aspect_bit(aspect_bit, src_image,
1632 region->srcSubresource.aspectMask) {
1633 enum isl_aux_usage src_aux_usage =
1634 anv_layout_to_aux_usage(&cmd_buffer->device->info, src_image,
1635 (1 << aspect_bit),
1636 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
1637 src_image_layout);
1638 enum isl_aux_usage dst_aux_usage =
1639 anv_layout_to_aux_usage(&cmd_buffer->device->info, dst_image,
1640 (1 << aspect_bit),
1641 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1642 dst_image_layout);
1643
1644 anv_image_msaa_resolve(cmd_buffer,
1645 src_image, src_aux_usage,
1646 region->srcSubresource.mipLevel,
1647 region->srcSubresource.baseArrayLayer,
1648 dst_image, dst_aux_usage,
1649 region->dstSubresource.mipLevel,
1650 region->dstSubresource.baseArrayLayer,
1651 (1 << aspect_bit),
1652 region->srcOffset.x,
1653 region->srcOffset.y,
1654 region->dstOffset.x,
1655 region->dstOffset.y,
1656 region->extent.width,
1657 region->extent.height,
1658 layer_count, BLORP_FILTER_NONE);
1659 }
1660 }
1661
anv_CmdResolveImage(VkCommandBuffer commandBuffer,VkImage srcImage,VkImageLayout srcImageLayout,VkImage dstImage,VkImageLayout dstImageLayout,uint32_t regionCount,const VkImageResolve * pRegions)1662 void anv_CmdResolveImage(
1663 VkCommandBuffer commandBuffer,
1664 VkImage srcImage,
1665 VkImageLayout srcImageLayout,
1666 VkImage dstImage,
1667 VkImageLayout dstImageLayout,
1668 uint32_t regionCount,
1669 const VkImageResolve* pRegions)
1670 {
1671 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1672 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
1673 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
1674
1675 assert(!src_image->format->can_ycbcr);
1676
1677 for (uint32_t r = 0; r < regionCount; r++) {
1678 VkImageResolve2KHR resolve = {
1679 .sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2_KHR,
1680 .srcSubresource = pRegions[r].srcSubresource,
1681 .srcOffset = pRegions[r].srcOffset,
1682 .dstSubresource = pRegions[r].dstSubresource,
1683 .dstOffset = pRegions[r].dstOffset,
1684 .extent = pRegions[r].extent,
1685 };
1686
1687 resolve_image(cmd_buffer,
1688 src_image, srcImageLayout,
1689 dst_image, dstImageLayout,
1690 &resolve);
1691 }
1692 }
1693
anv_CmdResolveImage2KHR(VkCommandBuffer commandBuffer,const VkResolveImageInfo2KHR * pResolveImageInfo)1694 void anv_CmdResolveImage2KHR(
1695 VkCommandBuffer commandBuffer,
1696 const VkResolveImageInfo2KHR* pResolveImageInfo)
1697 {
1698 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1699 ANV_FROM_HANDLE(anv_image, src_image, pResolveImageInfo->srcImage);
1700 ANV_FROM_HANDLE(anv_image, dst_image, pResolveImageInfo->dstImage);
1701
1702 assert(!src_image->format->can_ycbcr);
1703
1704 for (uint32_t r = 0; r < pResolveImageInfo->regionCount; r++) {
1705 resolve_image(cmd_buffer,
1706 src_image, pResolveImageInfo->srcImageLayout,
1707 dst_image, pResolveImageInfo->dstImageLayout,
1708 &pResolveImageInfo->pRegions[r]);
1709 }
1710 }
1711
1712 void
anv_image_copy_to_shadow(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,VkImageAspectFlagBits aspect,uint32_t base_level,uint32_t level_count,uint32_t base_layer,uint32_t layer_count)1713 anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
1714 const struct anv_image *image,
1715 VkImageAspectFlagBits aspect,
1716 uint32_t base_level, uint32_t level_count,
1717 uint32_t base_layer, uint32_t layer_count)
1718 {
1719 struct blorp_batch batch;
1720 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1721
1722 /* We don't know who touched the main surface last so flush a bunch of
1723 * caches to ensure we get good data.
1724 */
1725 cmd_buffer->state.pending_pipe_bits |=
1726 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT |
1727 ANV_PIPE_DATA_CACHE_FLUSH_BIT |
1728 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT |
1729 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
1730
1731 struct blorp_surf surf;
1732 get_blorp_surf_for_anv_image(cmd_buffer->device,
1733 image, aspect,
1734 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
1735 VK_IMAGE_LAYOUT_GENERAL,
1736 ISL_AUX_USAGE_NONE, &surf);
1737 assert(surf.aux_usage == ISL_AUX_USAGE_NONE);
1738
1739 struct blorp_surf shadow_surf;
1740 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
1741 image, aspect, &shadow_surf);
1742
1743 for (uint32_t l = 0; l < level_count; l++) {
1744 const uint32_t level = base_level + l;
1745
1746 const VkExtent3D extent = {
1747 .width = anv_minify(image->extent.width, level),
1748 .height = anv_minify(image->extent.height, level),
1749 .depth = anv_minify(image->extent.depth, level),
1750 };
1751
1752 if (image->type == VK_IMAGE_TYPE_3D)
1753 layer_count = extent.depth;
1754
1755 for (uint32_t a = 0; a < layer_count; a++) {
1756 const uint32_t layer = base_layer + a;
1757
1758 blorp_copy(&batch, &surf, level, layer,
1759 &shadow_surf, level, layer,
1760 0, 0, 0, 0, extent.width, extent.height);
1761 }
1762 }
1763
1764 /* We just wrote to the buffer with the render cache. Flush it. */
1765 cmd_buffer->state.pending_pipe_bits |=
1766 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
1767
1768 blorp_batch_finish(&batch);
1769 }
1770
1771 void
anv_image_clear_color(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,VkImageAspectFlagBits aspect,enum isl_aux_usage aux_usage,enum isl_format format,struct isl_swizzle swizzle,uint32_t level,uint32_t base_layer,uint32_t layer_count,VkRect2D area,union isl_color_value clear_color)1772 anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer,
1773 const struct anv_image *image,
1774 VkImageAspectFlagBits aspect,
1775 enum isl_aux_usage aux_usage,
1776 enum isl_format format, struct isl_swizzle swizzle,
1777 uint32_t level, uint32_t base_layer, uint32_t layer_count,
1778 VkRect2D area, union isl_color_value clear_color)
1779 {
1780 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1781
1782 /* We don't support planar images with multisampling yet */
1783 assert(image->n_planes == 1);
1784
1785 struct blorp_batch batch;
1786 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1787
1788 struct blorp_surf surf;
1789 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1790 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1791 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1792 aux_usage, &surf);
1793 anv_cmd_buffer_mark_image_written(cmd_buffer, image, aspect, aux_usage,
1794 level, base_layer, layer_count);
1795
1796 blorp_clear(&batch, &surf, format, anv_swizzle_for_render(swizzle),
1797 level, base_layer, layer_count,
1798 area.offset.x, area.offset.y,
1799 area.offset.x + area.extent.width,
1800 area.offset.y + area.extent.height,
1801 clear_color, NULL);
1802
1803 blorp_batch_finish(&batch);
1804 }
1805
1806 void
anv_image_clear_depth_stencil(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,VkImageAspectFlags aspects,enum isl_aux_usage depth_aux_usage,uint32_t level,uint32_t base_layer,uint32_t layer_count,VkRect2D area,float depth_value,uint8_t stencil_value)1807 anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer,
1808 const struct anv_image *image,
1809 VkImageAspectFlags aspects,
1810 enum isl_aux_usage depth_aux_usage,
1811 uint32_t level,
1812 uint32_t base_layer, uint32_t layer_count,
1813 VkRect2D area,
1814 float depth_value, uint8_t stencil_value)
1815 {
1816 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1817 VK_IMAGE_ASPECT_STENCIL_BIT));
1818
1819 struct blorp_batch batch;
1820 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1821
1822 struct blorp_surf depth = {};
1823 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1824 get_blorp_surf_for_anv_image(cmd_buffer->device,
1825 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1826 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1827 depth_aux_usage, &depth);
1828 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1829 }
1830
1831 struct blorp_surf stencil = {};
1832 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1833 uint32_t plane = anv_image_aspect_to_plane(image->aspects,
1834 VK_IMAGE_ASPECT_STENCIL_BIT);
1835 get_blorp_surf_for_anv_image(cmd_buffer->device,
1836 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1837 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1838 image->planes[plane].aux_usage, &stencil);
1839 }
1840
1841 /* Blorp may choose to clear stencil using RGBA32_UINT for better
1842 * performance. If it does this, we need to flush it out of the depth
1843 * cache before rendering to it.
1844 */
1845 cmd_buffer->state.pending_pipe_bits |=
1846 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1847
1848 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1849 level, base_layer, layer_count,
1850 area.offset.x, area.offset.y,
1851 area.offset.x + area.extent.width,
1852 area.offset.y + area.extent.height,
1853 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1854 depth_value,
1855 (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? 0xff : 0,
1856 stencil_value);
1857
1858 /* Blorp may choose to clear stencil using RGBA32_UINT for better
1859 * performance. If it does this, we need to flush it out of the render
1860 * cache before someone starts trying to do stencil on it.
1861 */
1862 cmd_buffer->state.pending_pipe_bits |=
1863 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1864
1865 struct blorp_surf stencil_shadow;
1866 if ((aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
1867 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
1868 VK_IMAGE_ASPECT_STENCIL_BIT,
1869 &stencil_shadow)) {
1870 union isl_color_value stencil_color = {
1871 .u32 = { stencil_value },
1872 };
1873 blorp_clear(&batch, &stencil_shadow,
1874 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1875 level, base_layer, layer_count,
1876 area.offset.x, area.offset.y,
1877 area.offset.x + area.extent.width,
1878 area.offset.y + area.extent.height,
1879 stencil_color, NULL);
1880 }
1881
1882 blorp_batch_finish(&batch);
1883 }
1884
1885 void
anv_image_hiz_op(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,VkImageAspectFlagBits aspect,uint32_t level,uint32_t base_layer,uint32_t layer_count,enum isl_aux_op hiz_op)1886 anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer,
1887 const struct anv_image *image,
1888 VkImageAspectFlagBits aspect, uint32_t level,
1889 uint32_t base_layer, uint32_t layer_count,
1890 enum isl_aux_op hiz_op)
1891 {
1892 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT);
1893 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level));
1894 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1895 assert(plane == 0);
1896
1897 struct blorp_batch batch;
1898 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1899
1900 struct blorp_surf surf;
1901 get_blorp_surf_for_anv_image(cmd_buffer->device,
1902 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1903 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1904 image->planes[plane].aux_usage, &surf);
1905 surf.clear_color.f32[0] = ANV_HZ_FC_VAL;
1906
1907 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count, hiz_op);
1908
1909 blorp_batch_finish(&batch);
1910 }
1911
1912 void
anv_image_hiz_clear(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,VkImageAspectFlags aspects,uint32_t level,uint32_t base_layer,uint32_t layer_count,VkRect2D area,uint8_t stencil_value)1913 anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer,
1914 const struct anv_image *image,
1915 VkImageAspectFlags aspects,
1916 uint32_t level,
1917 uint32_t base_layer, uint32_t layer_count,
1918 VkRect2D area, uint8_t stencil_value)
1919 {
1920 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1921 VK_IMAGE_ASPECT_STENCIL_BIT));
1922
1923 struct blorp_batch batch;
1924 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1925
1926 struct blorp_surf depth = {};
1927 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1928 uint32_t plane = anv_image_aspect_to_plane(image->aspects,
1929 VK_IMAGE_ASPECT_DEPTH_BIT);
1930 assert(base_layer + layer_count <=
1931 anv_image_aux_layers(image, VK_IMAGE_ASPECT_DEPTH_BIT, level));
1932 get_blorp_surf_for_anv_image(cmd_buffer->device,
1933 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1934 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1935 image->planes[plane].aux_usage, &depth);
1936 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1937 }
1938
1939 struct blorp_surf stencil = {};
1940 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1941 uint32_t plane = anv_image_aspect_to_plane(image->aspects,
1942 VK_IMAGE_ASPECT_STENCIL_BIT);
1943 get_blorp_surf_for_anv_image(cmd_buffer->device,
1944 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1945 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1946 image->planes[plane].aux_usage, &stencil);
1947 }
1948
1949 /* From the Sky Lake PRM Volume 7, "Depth Buffer Clear":
1950 *
1951 * "The following is required when performing a depth buffer clear with
1952 * using the WM_STATE or 3DSTATE_WM:
1953 *
1954 * * If other rendering operations have preceded this clear, a
1955 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
1956 * enabled must be issued before the rectangle primitive used for
1957 * the depth buffer clear operation.
1958 * * [...]"
1959 *
1960 * Even though the PRM only says that this is required if using 3DSTATE_WM
1961 * and a 3DPRIMITIVE, the GPU appears to also need this to avoid occasional
1962 * hangs when doing a clear with WM_HZ_OP.
1963 */
1964 cmd_buffer->state.pending_pipe_bits |=
1965 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1966
1967 blorp_hiz_clear_depth_stencil(&batch, &depth, &stencil,
1968 level, base_layer, layer_count,
1969 area.offset.x, area.offset.y,
1970 area.offset.x + area.extent.width,
1971 area.offset.y + area.extent.height,
1972 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1973 ANV_HZ_FC_VAL,
1974 aspects & VK_IMAGE_ASPECT_STENCIL_BIT,
1975 stencil_value);
1976
1977 blorp_batch_finish(&batch);
1978
1979 /* From the SKL PRM, Depth Buffer Clear:
1980 *
1981 * "Depth Buffer Clear Workaround
1982 *
1983 * Depth buffer clear pass using any of the methods (WM_STATE,
1984 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL
1985 * command with DEPTH_STALL bit and Depth FLUSH bits “set” before
1986 * starting to render. DepthStall and DepthFlush are not needed between
1987 * consecutive depth clear passes nor is it required if the depth-clear
1988 * pass was done with “full_surf_clear” bit set in the
1989 * 3DSTATE_WM_HZ_OP."
1990 *
1991 * Even though the PRM provides a bunch of conditions under which this is
1992 * supposedly unnecessary, we choose to perform the flush unconditionally
1993 * just to be safe.
1994 */
1995 cmd_buffer->state.pending_pipe_bits |=
1996 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1997 }
1998
1999 void
anv_image_mcs_op(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,enum isl_format format,struct isl_swizzle swizzle,VkImageAspectFlagBits aspect,uint32_t base_layer,uint32_t layer_count,enum isl_aux_op mcs_op,union isl_color_value * clear_value,bool predicate)2000 anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
2001 const struct anv_image *image,
2002 enum isl_format format, struct isl_swizzle swizzle,
2003 VkImageAspectFlagBits aspect,
2004 uint32_t base_layer, uint32_t layer_count,
2005 enum isl_aux_op mcs_op, union isl_color_value *clear_value,
2006 bool predicate)
2007 {
2008 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
2009 assert(image->samples > 1);
2010 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0));
2011
2012 /* Multisampling with multi-planar formats is not supported */
2013 assert(image->n_planes == 1);
2014
2015 struct blorp_batch batch;
2016 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
2017 BLORP_BATCH_PREDICATE_ENABLE * predicate +
2018 BLORP_BATCH_NO_UPDATE_CLEAR_COLOR * !clear_value);
2019
2020 struct blorp_surf surf;
2021 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
2022 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
2023 ISL_AUX_USAGE_MCS, &surf);
2024
2025 /* Blorp will store the clear color for us if we provide the clear color
2026 * address and we are doing a fast clear. So we save the clear value into
2027 * the blorp surface.
2028 */
2029 if (clear_value)
2030 surf.clear_color = *clear_value;
2031
2032 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
2033 *
2034 * "After Render target fast clear, pipe-control with color cache
2035 * write-flush must be issued before sending any DRAW commands on
2036 * that render target."
2037 *
2038 * This comment is a bit cryptic and doesn't really tell you what's going
2039 * or what's really needed. It appears that fast clear ops are not
2040 * properly synchronized with other drawing. This means that we cannot
2041 * have a fast clear operation in the pipe at the same time as other
2042 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
2043 * that the contents of the previous draw hit the render target before we
2044 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
2045 * that it is completed before any additional drawing occurs.
2046 */
2047 cmd_buffer->state.pending_pipe_bits |=
2048 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
2049
2050 switch (mcs_op) {
2051 case ISL_AUX_OP_FAST_CLEAR:
2052 blorp_fast_clear(&batch, &surf, format, swizzle,
2053 0, base_layer, layer_count,
2054 0, 0, image->extent.width, image->extent.height);
2055 break;
2056 case ISL_AUX_OP_PARTIAL_RESOLVE:
2057 blorp_mcs_partial_resolve(&batch, &surf, format,
2058 base_layer, layer_count);
2059 break;
2060 case ISL_AUX_OP_FULL_RESOLVE:
2061 case ISL_AUX_OP_AMBIGUATE:
2062 default:
2063 unreachable("Unsupported MCS operation");
2064 }
2065
2066 cmd_buffer->state.pending_pipe_bits |=
2067 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
2068
2069 blorp_batch_finish(&batch);
2070 }
2071
2072 void
anv_image_ccs_op(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,enum isl_format format,struct isl_swizzle swizzle,VkImageAspectFlagBits aspect,uint32_t level,uint32_t base_layer,uint32_t layer_count,enum isl_aux_op ccs_op,union isl_color_value * clear_value,bool predicate)2073 anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
2074 const struct anv_image *image,
2075 enum isl_format format, struct isl_swizzle swizzle,
2076 VkImageAspectFlagBits aspect, uint32_t level,
2077 uint32_t base_layer, uint32_t layer_count,
2078 enum isl_aux_op ccs_op, union isl_color_value *clear_value,
2079 bool predicate)
2080 {
2081 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
2082 assert(image->samples == 1);
2083 assert(level < anv_image_aux_levels(image, aspect));
2084 /* Multi-LOD YcBcR is not allowed */
2085 assert(image->n_planes == 1 || level == 0);
2086 assert(base_layer + layer_count <=
2087 anv_image_aux_layers(image, aspect, level));
2088
2089 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
2090 uint32_t width_div = image->format->planes[plane].denominator_scales[0];
2091 uint32_t height_div = image->format->planes[plane].denominator_scales[1];
2092 uint32_t level_width = anv_minify(image->extent.width, level) / width_div;
2093 uint32_t level_height = anv_minify(image->extent.height, level) / height_div;
2094
2095 struct blorp_batch batch;
2096 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
2097 BLORP_BATCH_PREDICATE_ENABLE * predicate +
2098 BLORP_BATCH_NO_UPDATE_CLEAR_COLOR * !clear_value);
2099
2100 struct blorp_surf surf;
2101 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
2102 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
2103 image->planes[plane].aux_usage,
2104 &surf);
2105
2106 /* Blorp will store the clear color for us if we provide the clear color
2107 * address and we are doing a fast clear. So we save the clear value into
2108 * the blorp surface.
2109 */
2110 if (clear_value)
2111 surf.clear_color = *clear_value;
2112
2113 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
2114 *
2115 * "After Render target fast clear, pipe-control with color cache
2116 * write-flush must be issued before sending any DRAW commands on
2117 * that render target."
2118 *
2119 * This comment is a bit cryptic and doesn't really tell you what's going
2120 * or what's really needed. It appears that fast clear ops are not
2121 * properly synchronized with other drawing. This means that we cannot
2122 * have a fast clear operation in the pipe at the same time as other
2123 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
2124 * that the contents of the previous draw hit the render target before we
2125 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
2126 * that it is completed before any additional drawing occurs.
2127 */
2128 cmd_buffer->state.pending_pipe_bits |=
2129 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
2130
2131 switch (ccs_op) {
2132 case ISL_AUX_OP_FAST_CLEAR:
2133 blorp_fast_clear(&batch, &surf, format, swizzle,
2134 level, base_layer, layer_count,
2135 0, 0, level_width, level_height);
2136 break;
2137 case ISL_AUX_OP_FULL_RESOLVE:
2138 case ISL_AUX_OP_PARTIAL_RESOLVE:
2139 blorp_ccs_resolve(&batch, &surf, level, base_layer, layer_count,
2140 format, ccs_op);
2141 break;
2142 case ISL_AUX_OP_AMBIGUATE:
2143 for (uint32_t a = 0; a < layer_count; a++) {
2144 const uint32_t layer = base_layer + a;
2145 blorp_ccs_ambiguate(&batch, &surf, level, layer);
2146 }
2147 break;
2148 default:
2149 unreachable("Unsupported CCS operation");
2150 }
2151
2152 cmd_buffer->state.pending_pipe_bits |=
2153 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
2154
2155 blorp_batch_finish(&batch);
2156 }
2157