1 /*
2  * Copyright © 2019 Red Hat.
3  * Copyright © 2022 Collabora, LTD
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice (including the next
13  * paragraph) shall be included in all copies or substantial portions of the
14  * Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22  * IN THE SOFTWARE.
23  */
24 
25 #include "vk_alloc.h"
26 #include "vk_cmd_enqueue_entrypoints.h"
27 #include "vk_command_buffer.h"
28 #include "vk_device.h"
29 #include "vk_pipeline_layout.h"
30 #include "vk_util.h"
31 
32 VKAPI_ATTR void VKAPI_CALL
vk_cmd_enqueue_CmdDrawMultiEXT(VkCommandBuffer commandBuffer,uint32_t drawCount,const VkMultiDrawInfoEXT * pVertexInfo,uint32_t instanceCount,uint32_t firstInstance,uint32_t stride)33 vk_cmd_enqueue_CmdDrawMultiEXT(VkCommandBuffer commandBuffer,
34                                uint32_t drawCount,
35                                const VkMultiDrawInfoEXT *pVertexInfo,
36                                uint32_t instanceCount,
37                                uint32_t firstInstance,
38                                uint32_t stride)
39 {
40    VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
41 
42    struct vk_cmd_queue_entry *cmd =
43       vk_zalloc(cmd_buffer->cmd_queue.alloc, sizeof(*cmd), 8,
44                 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
45    if (!cmd)
46       return;
47 
48    cmd->type = VK_CMD_DRAW_MULTI_EXT;
49    list_addtail(&cmd->cmd_link, &cmd_buffer->cmd_queue.cmds);
50 
51    cmd->u.draw_multi_ext.draw_count = drawCount;
52    if (pVertexInfo) {
53       unsigned i = 0;
54       cmd->u.draw_multi_ext.vertex_info =
55          vk_zalloc(cmd_buffer->cmd_queue.alloc,
56                    sizeof(*cmd->u.draw_multi_ext.vertex_info) * drawCount, 8,
57                    VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
58 
59       vk_foreach_multi_draw(draw, i, pVertexInfo, drawCount, stride) {
60          memcpy(&cmd->u.draw_multi_ext.vertex_info[i], draw,
61                 sizeof(*cmd->u.draw_multi_ext.vertex_info));
62       }
63    }
64    cmd->u.draw_multi_ext.instance_count = instanceCount;
65    cmd->u.draw_multi_ext.first_instance = firstInstance;
66    cmd->u.draw_multi_ext.stride = stride;
67 }
68 
69 VKAPI_ATTR void VKAPI_CALL
vk_cmd_enqueue_CmdDrawMultiIndexedEXT(VkCommandBuffer commandBuffer,uint32_t drawCount,const VkMultiDrawIndexedInfoEXT * pIndexInfo,uint32_t instanceCount,uint32_t firstInstance,uint32_t stride,const int32_t * pVertexOffset)70 vk_cmd_enqueue_CmdDrawMultiIndexedEXT(VkCommandBuffer commandBuffer,
71                                       uint32_t drawCount,
72                                       const VkMultiDrawIndexedInfoEXT *pIndexInfo,
73                                       uint32_t instanceCount,
74                                       uint32_t firstInstance,
75                                       uint32_t stride,
76                                       const int32_t *pVertexOffset)
77 {
78    VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
79 
80    struct vk_cmd_queue_entry *cmd =
81       vk_zalloc(cmd_buffer->cmd_queue.alloc, sizeof(*cmd), 8,
82                 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
83    if (!cmd)
84       return;
85 
86    cmd->type = VK_CMD_DRAW_MULTI_INDEXED_EXT;
87    list_addtail(&cmd->cmd_link, &cmd_buffer->cmd_queue.cmds);
88 
89    cmd->u.draw_multi_indexed_ext.draw_count = drawCount;
90 
91    if (pIndexInfo) {
92       unsigned i = 0;
93       cmd->u.draw_multi_indexed_ext.index_info =
94          vk_zalloc(cmd_buffer->cmd_queue.alloc,
95                    sizeof(*cmd->u.draw_multi_indexed_ext.index_info) * drawCount, 8,
96                    VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
97 
98       vk_foreach_multi_draw_indexed(draw, i, pIndexInfo, drawCount, stride) {
99          cmd->u.draw_multi_indexed_ext.index_info[i].firstIndex = draw->firstIndex;
100          cmd->u.draw_multi_indexed_ext.index_info[i].indexCount = draw->indexCount;
101          if (pVertexOffset == NULL)
102             cmd->u.draw_multi_indexed_ext.index_info[i].vertexOffset = draw->vertexOffset;
103       }
104    }
105 
106    cmd->u.draw_multi_indexed_ext.instance_count = instanceCount;
107    cmd->u.draw_multi_indexed_ext.first_instance = firstInstance;
108    cmd->u.draw_multi_indexed_ext.stride = stride;
109 
110    if (pVertexOffset) {
111       cmd->u.draw_multi_indexed_ext.vertex_offset =
112          vk_zalloc(cmd_buffer->cmd_queue.alloc,
113                    sizeof(*cmd->u.draw_multi_indexed_ext.vertex_offset), 8,
114                    VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
115 
116       memcpy(cmd->u.draw_multi_indexed_ext.vertex_offset, pVertexOffset,
117              sizeof(*cmd->u.draw_multi_indexed_ext.vertex_offset));
118    }
119 }
120 
121 static void
push_descriptors_set_free(struct vk_cmd_queue * queue,struct vk_cmd_queue_entry * cmd)122 push_descriptors_set_free(struct vk_cmd_queue *queue,
123                           struct vk_cmd_queue_entry *cmd)
124 {
125   struct vk_cmd_push_descriptor_set_khr *pds = &cmd->u.push_descriptor_set_khr;
126   for (unsigned i = 0; i < pds->descriptor_write_count; i++) {
127     VkWriteDescriptorSet *entry = &pds->descriptor_writes[i];
128     switch (entry->descriptorType) {
129     case VK_DESCRIPTOR_TYPE_SAMPLER:
130     case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
131     case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
132     case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
133     case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
134        vk_free(queue->alloc, (void *)entry->pImageInfo);
135        break;
136     case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
137     case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
138        vk_free(queue->alloc, (void *)entry->pTexelBufferView);
139        break;
140     case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
141     case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
142     case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
143     case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
144     default:
145        vk_free(queue->alloc, (void *)entry->pBufferInfo);
146        break;
147     }
148   }
149 }
150 
151 VKAPI_ATTR void VKAPI_CALL
vk_cmd_enqueue_CmdPushDescriptorSetKHR(VkCommandBuffer commandBuffer,VkPipelineBindPoint pipelineBindPoint,VkPipelineLayout layout,uint32_t set,uint32_t descriptorWriteCount,const VkWriteDescriptorSet * pDescriptorWrites)152 vk_cmd_enqueue_CmdPushDescriptorSetKHR(VkCommandBuffer commandBuffer,
153                                        VkPipelineBindPoint pipelineBindPoint,
154                                        VkPipelineLayout layout,
155                                        uint32_t set,
156                                        uint32_t descriptorWriteCount,
157                                        const VkWriteDescriptorSet *pDescriptorWrites)
158 {
159    VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
160    struct vk_cmd_push_descriptor_set_khr *pds;
161 
162    struct vk_cmd_queue_entry *cmd =
163       vk_zalloc(cmd_buffer->cmd_queue.alloc, sizeof(*cmd), 8,
164                 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
165    if (!cmd)
166       return;
167 
168    pds = &cmd->u.push_descriptor_set_khr;
169 
170    cmd->type = VK_CMD_PUSH_DESCRIPTOR_SET_KHR;
171    cmd->driver_free_cb = push_descriptors_set_free;
172    list_addtail(&cmd->cmd_link, &cmd_buffer->cmd_queue.cmds);
173 
174    pds->pipeline_bind_point = pipelineBindPoint;
175    pds->layout = layout;
176    pds->set = set;
177    pds->descriptor_write_count = descriptorWriteCount;
178 
179    if (pDescriptorWrites) {
180       pds->descriptor_writes =
181          vk_zalloc(cmd_buffer->cmd_queue.alloc,
182                    sizeof(*pds->descriptor_writes) * descriptorWriteCount, 8,
183                    VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
184       memcpy(pds->descriptor_writes,
185              pDescriptorWrites,
186              sizeof(*pds->descriptor_writes) * descriptorWriteCount);
187 
188       for (unsigned i = 0; i < descriptorWriteCount; i++) {
189          switch (pds->descriptor_writes[i].descriptorType) {
190          case VK_DESCRIPTOR_TYPE_SAMPLER:
191          case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
192          case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
193          case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
194          case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
195             pds->descriptor_writes[i].pImageInfo =
196                vk_zalloc(cmd_buffer->cmd_queue.alloc,
197                          sizeof(VkDescriptorImageInfo) * pds->descriptor_writes[i].descriptorCount, 8,
198                          VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
199             memcpy((VkDescriptorImageInfo *)pds->descriptor_writes[i].pImageInfo,
200                    pDescriptorWrites[i].pImageInfo,
201                    sizeof(VkDescriptorImageInfo) * pds->descriptor_writes[i].descriptorCount);
202             break;
203          case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
204          case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
205             pds->descriptor_writes[i].pTexelBufferView =
206                vk_zalloc(cmd_buffer->cmd_queue.alloc,
207                          sizeof(VkBufferView) * pds->descriptor_writes[i].descriptorCount, 8,
208                          VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
209             memcpy((VkBufferView *)pds->descriptor_writes[i].pTexelBufferView,
210                    pDescriptorWrites[i].pTexelBufferView,
211                    sizeof(VkBufferView) * pds->descriptor_writes[i].descriptorCount);
212             break;
213          case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
214          case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
215          case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
216          case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
217          default:
218             pds->descriptor_writes[i].pBufferInfo =
219                vk_zalloc(cmd_buffer->cmd_queue.alloc,
220                          sizeof(VkDescriptorBufferInfo) * pds->descriptor_writes[i].descriptorCount, 8,
221                          VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
222             memcpy((VkDescriptorBufferInfo *)pds->descriptor_writes[i].pBufferInfo,
223                    pDescriptorWrites[i].pBufferInfo,
224                    sizeof(VkDescriptorBufferInfo) * pds->descriptor_writes[i].descriptorCount);
225             break;
226          }
227       }
228    }
229 }
230 
231 static void
unref_pipeline_layout(struct vk_cmd_queue * queue,struct vk_cmd_queue_entry * cmd)232 unref_pipeline_layout(struct vk_cmd_queue *queue,
233                       struct vk_cmd_queue_entry *cmd)
234 {
235    struct vk_command_buffer *cmd_buffer =
236       container_of(queue, struct vk_command_buffer, cmd_queue);
237    VK_FROM_HANDLE(vk_pipeline_layout, layout,
238                   cmd->u.bind_descriptor_sets.layout);
239 
240    assert(cmd->type == VK_CMD_BIND_DESCRIPTOR_SETS);
241 
242    vk_pipeline_layout_unref(cmd_buffer->base.device, layout);
243 }
244 
245 VKAPI_ATTR void VKAPI_CALL
vk_cmd_enqueue_CmdBindDescriptorSets(VkCommandBuffer commandBuffer,VkPipelineBindPoint pipelineBindPoint,VkPipelineLayout layout,uint32_t firstSet,uint32_t descriptorSetCount,const VkDescriptorSet * pDescriptorSets,uint32_t dynamicOffsetCount,const uint32_t * pDynamicOffsets)246 vk_cmd_enqueue_CmdBindDescriptorSets(VkCommandBuffer commandBuffer,
247                                      VkPipelineBindPoint pipelineBindPoint,
248                                      VkPipelineLayout layout,
249                                      uint32_t firstSet,
250                                      uint32_t descriptorSetCount,
251                                      const VkDescriptorSet* pDescriptorSets,
252                                      uint32_t dynamicOffsetCount,
253                                      const uint32_t *pDynamicOffsets)
254 {
255    VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
256 
257    struct vk_cmd_queue_entry *cmd =
258       vk_zalloc(cmd_buffer->cmd_queue.alloc, sizeof(*cmd), 8,
259                 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
260    if (!cmd)
261       return;
262 
263    cmd->type = VK_CMD_BIND_DESCRIPTOR_SETS;
264    list_addtail(&cmd->cmd_link, &cmd_buffer->cmd_queue.cmds);
265 
266    /* We need to hold a reference to the descriptor set as long as this
267     * command is in the queue.  Otherwise, it may get deleted out from under
268     * us before the command is replayed.
269     */
270    vk_pipeline_layout_ref(vk_pipeline_layout_from_handle(layout));
271    cmd->u.bind_descriptor_sets.layout = layout;
272    cmd->driver_free_cb = unref_pipeline_layout;
273 
274    cmd->u.bind_descriptor_sets.pipeline_bind_point = pipelineBindPoint;
275    cmd->u.bind_descriptor_sets.first_set = firstSet;
276    cmd->u.bind_descriptor_sets.descriptor_set_count = descriptorSetCount;
277    if (pDescriptorSets) {
278       cmd->u.bind_descriptor_sets.descriptor_sets =
279          vk_zalloc(cmd_buffer->cmd_queue.alloc,
280                    sizeof(*cmd->u.bind_descriptor_sets.descriptor_sets) * descriptorSetCount, 8,
281                    VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
282 
283       memcpy(cmd->u.bind_descriptor_sets.descriptor_sets, pDescriptorSets,
284              sizeof(*cmd->u.bind_descriptor_sets.descriptor_sets) * descriptorSetCount);
285    }
286    cmd->u.bind_descriptor_sets.dynamic_offset_count = dynamicOffsetCount;
287    if (pDynamicOffsets) {
288       cmd->u.bind_descriptor_sets.dynamic_offsets =
289          vk_zalloc(cmd_buffer->cmd_queue.alloc,
290                    sizeof(*cmd->u.bind_descriptor_sets.dynamic_offsets) * dynamicOffsetCount, 8,
291                    VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
292 
293       memcpy(cmd->u.bind_descriptor_sets.dynamic_offsets, pDynamicOffsets,
294              sizeof(*cmd->u.bind_descriptor_sets.dynamic_offsets) * dynamicOffsetCount);
295    }
296 }
297 
298 #ifdef VK_ENABLE_BETA_EXTENSIONS
299 static void
dispatch_graph_amdx_free(struct vk_cmd_queue * queue,struct vk_cmd_queue_entry * cmd)300 dispatch_graph_amdx_free(struct vk_cmd_queue *queue, struct vk_cmd_queue_entry *cmd)
301 {
302    VkDispatchGraphCountInfoAMDX *count_info = cmd->u.dispatch_graph_amdx.count_info;
303    void *infos = (void *)count_info->infos.hostAddress;
304 
305    for (uint32_t i = 0; i < count_info->count; i++) {
306       VkDispatchGraphInfoAMDX *info = (void *)((const uint8_t *)infos + i * count_info->stride);
307       vk_free(queue->alloc, (void *)info->payloads.hostAddress);
308    }
309 
310    vk_free(queue->alloc, infos);
311 }
312 
313 VKAPI_ATTR void VKAPI_CALL
vk_cmd_enqueue_CmdDispatchGraphAMDX(VkCommandBuffer commandBuffer,VkDeviceAddress scratch,const VkDispatchGraphCountInfoAMDX * pCountInfo)314 vk_cmd_enqueue_CmdDispatchGraphAMDX(VkCommandBuffer commandBuffer, VkDeviceAddress scratch,
315                                     const VkDispatchGraphCountInfoAMDX *pCountInfo)
316 {
317    VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
318 
319    if (vk_command_buffer_has_error(cmd_buffer))
320       return;
321 
322    VkResult result = VK_SUCCESS;
323    const VkAllocationCallbacks *alloc = cmd_buffer->cmd_queue.alloc;
324 
325    struct vk_cmd_queue_entry *cmd =
326       vk_zalloc(alloc, sizeof(struct vk_cmd_queue_entry), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
327    if (!cmd) {
328       result = VK_ERROR_OUT_OF_HOST_MEMORY;
329       goto err;
330    }
331 
332    cmd->type = VK_CMD_DISPATCH_GRAPH_AMDX;
333    cmd->driver_free_cb = dispatch_graph_amdx_free;
334 
335    cmd->u.dispatch_graph_amdx.scratch = scratch;
336 
337    cmd->u.dispatch_graph_amdx.count_info =
338       vk_zalloc(alloc, sizeof(VkDispatchGraphCountInfoAMDX), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
339    if (cmd->u.dispatch_graph_amdx.count_info == NULL)
340       goto err;
341 
342    memcpy((void *)cmd->u.dispatch_graph_amdx.count_info, pCountInfo,
343           sizeof(VkDispatchGraphCountInfoAMDX));
344 
345    uint32_t infos_size = pCountInfo->count * pCountInfo->stride;
346    void *infos = vk_zalloc(alloc, infos_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
347    cmd->u.dispatch_graph_amdx.count_info->infos.hostAddress = infos;
348    memcpy(infos, pCountInfo->infos.hostAddress, infos_size);
349 
350    for (uint32_t i = 0; i < pCountInfo->count; i++) {
351       VkDispatchGraphInfoAMDX *info = (void *)((const uint8_t *)infos + i * pCountInfo->stride);
352 
353       uint32_t payloads_size = info->payloadCount * info->payloadStride;
354       void *dst_payload = vk_zalloc(alloc, payloads_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
355       memcpy(dst_payload, info->payloads.hostAddress, payloads_size);
356       info->payloads.hostAddress = dst_payload;
357    }
358 
359    list_addtail(&cmd->cmd_link, &cmd_buffer->cmd_queue.cmds);
360    goto finish;
361 err:
362    if (cmd) {
363       vk_free(alloc, cmd);
364       dispatch_graph_amdx_free(&cmd_buffer->cmd_queue, cmd);
365    }
366 
367 finish:
368    if (unlikely(result != VK_SUCCESS))
369       vk_command_buffer_set_error(cmd_buffer, result);
370 }
371 #endif
372