1 /*
2 * Permission is hereby granted, free of charge, to any person obtaining a
3 * copy of this software and associated documentation files (the "Software"),
4 * to deal in the Software without restriction, including without limitation
5 * on the rights to use, copy, modify, merge, publish, distribute, sub
6 * license, and/or sell copies of the Software, and to permit persons to whom
7 * the Software is furnished to do so, subject to the following conditions:
8 *
9 * The above copyright notice and this permission notice (including the next
10 * paragraph) shall be included in all copies or substantial portions of the
11 * Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * Authors:
22 * Adam Rak <adam.rak@streamnovation.com>
23 */
24
25 #include "pipe/p_defines.h"
26 #include "pipe/p_state.h"
27 #include "pipe/p_context.h"
28 #include "util/u_blitter.h"
29 #include "util/list.h"
30 #include "util/u_transfer.h"
31 #include "util/u_surface.h"
32 #include "util/u_pack_color.h"
33 #include "util/u_math.h"
34 #include "util/u_memory.h"
35 #include "util/u_inlines.h"
36 #include "util/u_framebuffer.h"
37 #include "r600_shader.h"
38 #include "r600_pipe.h"
39 #include "r600_formats.h"
40 #include "compute_memory_pool.h"
41 #include "evergreen_compute.h"
42 #include "evergreen_compute_internal.h"
43 #include <inttypes.h>
44
45 #define ITEM_ALIGNMENT 1024
46
47 /* A few forward declarations of static functions */
48 static void compute_memory_shadow(struct compute_memory_pool* pool,
49 struct pipe_context *pipe, int device_to_host);
50
51 static void compute_memory_defrag(struct compute_memory_pool *pool,
52 struct pipe_resource *src, struct pipe_resource *dst,
53 struct pipe_context *pipe);
54
55 static int compute_memory_promote_item(struct compute_memory_pool *pool,
56 struct compute_memory_item *item, struct pipe_context *pipe,
57 int64_t allocated);
58
59 static void compute_memory_move_item(struct compute_memory_pool *pool,
60 struct pipe_resource *src, struct pipe_resource *dst,
61 struct compute_memory_item *item, uint64_t new_start_in_dw,
62 struct pipe_context *pipe);
63
64 static void compute_memory_transfer(struct compute_memory_pool* pool,
65 struct pipe_context * pipe, int device_to_host,
66 struct compute_memory_item* chunk, void* data,
67 int offset_in_chunk, int size);
68
69 /**
70 * Creates a new pool.
71 */
compute_memory_pool_new(struct r600_screen * rscreen)72 struct compute_memory_pool* compute_memory_pool_new(
73 struct r600_screen * rscreen)
74 {
75 struct compute_memory_pool* pool = (struct compute_memory_pool*)
76 CALLOC(sizeof(struct compute_memory_pool), 1);
77 if (!pool)
78 return NULL;
79
80 COMPUTE_DBG(rscreen, "* compute_memory_pool_new()\n");
81
82 pool->screen = rscreen;
83 pool->item_list = (struct list_head *)
84 CALLOC(sizeof(struct list_head), 1);
85 pool->unallocated_list = (struct list_head *)
86 CALLOC(sizeof(struct list_head), 1);
87 list_inithead(pool->item_list);
88 list_inithead(pool->unallocated_list);
89 return pool;
90 }
91
92 /**
93 * Initializes the pool with a size of \a initial_size_in_dw.
94 * \param pool The pool to be initialized.
95 * \param initial_size_in_dw The initial size.
96 * \see compute_memory_grow_defrag_pool
97 */
compute_memory_pool_init(struct compute_memory_pool * pool,unsigned initial_size_in_dw)98 static void compute_memory_pool_init(struct compute_memory_pool * pool,
99 unsigned initial_size_in_dw)
100 {
101
102 COMPUTE_DBG(pool->screen, "* compute_memory_pool_init() initial_size_in_dw = %u\n",
103 initial_size_in_dw);
104
105 pool->size_in_dw = initial_size_in_dw;
106 pool->bo = r600_compute_buffer_alloc_vram(pool->screen,
107 pool->size_in_dw * 4);
108 }
109
110 /**
111 * Frees all stuff in the pool and the pool struct itself too.
112 */
compute_memory_pool_delete(struct compute_memory_pool * pool)113 void compute_memory_pool_delete(struct compute_memory_pool* pool)
114 {
115 COMPUTE_DBG(pool->screen, "* compute_memory_pool_delete()\n");
116 free(pool->shadow);
117 r600_resource_reference(&pool->bo, NULL);
118 /* In theory, all of the items were freed in compute_memory_free.
119 * Just delete the list heads
120 */
121 free(pool->item_list);
122 free(pool->unallocated_list);
123 /* And then the pool itself */
124 free(pool);
125 }
126
127 /**
128 * Reallocates and defragments the pool, conserves data.
129 * \returns -1 if it fails, 0 otherwise
130 * \see compute_memory_finalize_pending
131 */
compute_memory_grow_defrag_pool(struct compute_memory_pool * pool,struct pipe_context * pipe,int new_size_in_dw)132 static int compute_memory_grow_defrag_pool(struct compute_memory_pool *pool,
133 struct pipe_context *pipe, int new_size_in_dw)
134 {
135 new_size_in_dw = align(new_size_in_dw, ITEM_ALIGNMENT);
136
137 COMPUTE_DBG(pool->screen, "* compute_memory_grow_defrag_pool() "
138 "new_size_in_dw = %d (%d bytes)\n",
139 new_size_in_dw, new_size_in_dw * 4);
140
141 assert(new_size_in_dw >= pool->size_in_dw);
142
143 if (!pool->bo) {
144 compute_memory_pool_init(pool, MAX2(new_size_in_dw, 1024 * 16));
145 } else {
146 struct r600_resource *temp = NULL;
147
148 temp = r600_compute_buffer_alloc_vram(pool->screen, new_size_in_dw * 4);
149
150 if (temp != NULL) {
151 struct pipe_resource *src = (struct pipe_resource *)pool->bo;
152 struct pipe_resource *dst = (struct pipe_resource *)temp;
153
154 COMPUTE_DBG(pool->screen, " Growing and defragmenting the pool "
155 "using a temporary resource\n");
156
157 compute_memory_defrag(pool, src, dst, pipe);
158
159 /* Release the old buffer */
160 r600_resource_reference(&pool->bo, NULL);
161 pool->bo = temp;
162 pool->size_in_dw = new_size_in_dw;
163 }
164 else {
165 COMPUTE_DBG(pool->screen, " The creation of the temporary resource failed\n"
166 " Falling back to using 'shadow'\n");
167
168 compute_memory_shadow(pool, pipe, 1);
169 pool->shadow = realloc(pool->shadow, new_size_in_dw * 4);
170 if (pool->shadow == NULL)
171 return -1;
172
173 pool->size_in_dw = new_size_in_dw;
174 /* Release the old buffer */
175 r600_resource_reference(&pool->bo, NULL);
176 pool->bo = r600_compute_buffer_alloc_vram(pool->screen, pool->size_in_dw * 4);
177 compute_memory_shadow(pool, pipe, 0);
178
179 if (pool->status & POOL_FRAGMENTED) {
180 struct pipe_resource *src = (struct pipe_resource *)pool->bo;
181 compute_memory_defrag(pool, src, src, pipe);
182 }
183 }
184 }
185
186 return 0;
187 }
188
189 /**
190 * Copy pool from device to host, or host to device.
191 * \param device_to_host 1 for device->host, 0 for host->device
192 * \see compute_memory_grow_defrag_pool
193 */
compute_memory_shadow(struct compute_memory_pool * pool,struct pipe_context * pipe,int device_to_host)194 static void compute_memory_shadow(struct compute_memory_pool* pool,
195 struct pipe_context * pipe, int device_to_host)
196 {
197 struct compute_memory_item chunk;
198
199 COMPUTE_DBG(pool->screen, "* compute_memory_shadow() device_to_host = %d\n",
200 device_to_host);
201
202 chunk.id = 0;
203 chunk.start_in_dw = 0;
204 chunk.size_in_dw = pool->size_in_dw;
205 compute_memory_transfer(pool, pipe, device_to_host, &chunk,
206 pool->shadow, 0, pool->size_in_dw*4);
207 }
208
209 /**
210 * Moves all the items marked for promotion from the \a unallocated_list
211 * to the \a item_list.
212 * \return -1 if it fails, 0 otherwise
213 * \see evergreen_set_global_binding
214 */
compute_memory_finalize_pending(struct compute_memory_pool * pool,struct pipe_context * pipe)215 int compute_memory_finalize_pending(struct compute_memory_pool* pool,
216 struct pipe_context * pipe)
217 {
218 struct compute_memory_item *item, *next;
219
220 int64_t allocated = 0;
221 int64_t unallocated = 0;
222 int64_t last_pos;
223
224 int err = 0;
225
226 COMPUTE_DBG(pool->screen, "* compute_memory_finalize_pending()\n");
227
228 LIST_FOR_EACH_ENTRY(item, pool->item_list, link) {
229 COMPUTE_DBG(pool->screen, " + list: offset = %"PRIi64" id = %"PRIi64" size = %"PRIi64" "
230 "(%"PRIi64" bytes)\n", item->start_in_dw, item->id,
231 item->size_in_dw, item->size_in_dw * 4);
232 }
233
234 /* Calculate the total allocated size */
235 LIST_FOR_EACH_ENTRY(item, pool->item_list, link) {
236 allocated += align(item->size_in_dw, ITEM_ALIGNMENT);
237 }
238
239 /* Calculate the total unallocated size of the items that
240 * will be promoted to the pool */
241 LIST_FOR_EACH_ENTRY(item, pool->unallocated_list, link) {
242 if (item->status & ITEM_FOR_PROMOTING)
243 unallocated += align(item->size_in_dw, ITEM_ALIGNMENT);
244 }
245
246 if (unallocated == 0) {
247 return 0;
248 }
249
250 if (pool->size_in_dw < allocated + unallocated) {
251 err = compute_memory_grow_defrag_pool(pool, pipe, allocated + unallocated);
252 if (err == -1)
253 return -1;
254 }
255 else if (pool->status & POOL_FRAGMENTED) {
256 struct pipe_resource *src = (struct pipe_resource *)pool->bo;
257 compute_memory_defrag(pool, src, src, pipe);
258 }
259
260 /* After defragmenting the pool, allocated is equal to the first available
261 * position for new items in the pool */
262 last_pos = allocated;
263
264 /* Loop through all the unallocated items, check if they are marked
265 * for promoting, allocate space for them and add them to the item_list. */
266 LIST_FOR_EACH_ENTRY_SAFE(item, next, pool->unallocated_list, link) {
267 if (item->status & ITEM_FOR_PROMOTING) {
268 err = compute_memory_promote_item(pool, item, pipe, last_pos);
269 item->status &= ~ITEM_FOR_PROMOTING;
270
271 last_pos += align(item->size_in_dw, ITEM_ALIGNMENT);
272
273 if (err == -1)
274 return -1;
275 }
276 }
277
278 return 0;
279 }
280
281 /**
282 * Defragments the pool, so that there's no gap between items.
283 * \param pool The pool to be defragmented
284 * \param src The origin resource
285 * \param dst The destination resource
286 * \see compute_memory_grow_defrag_pool and compute_memory_finalize_pending
287 */
compute_memory_defrag(struct compute_memory_pool * pool,struct pipe_resource * src,struct pipe_resource * dst,struct pipe_context * pipe)288 static void compute_memory_defrag(struct compute_memory_pool *pool,
289 struct pipe_resource *src, struct pipe_resource *dst,
290 struct pipe_context *pipe)
291 {
292 struct compute_memory_item *item;
293 int64_t last_pos;
294
295 COMPUTE_DBG(pool->screen, "* compute_memory_defrag()\n");
296
297 last_pos = 0;
298 LIST_FOR_EACH_ENTRY(item, pool->item_list, link) {
299 if (src != dst || item->start_in_dw != last_pos) {
300 assert(last_pos <= item->start_in_dw);
301
302 compute_memory_move_item(pool, src, dst,
303 item, last_pos, pipe);
304 }
305
306 last_pos += align(item->size_in_dw, ITEM_ALIGNMENT);
307 }
308
309 pool->status &= ~POOL_FRAGMENTED;
310 }
311
312 /**
313 * Moves an item from the \a unallocated_list to the \a item_list.
314 * \param item The item that will be promoted.
315 * \return -1 if it fails, 0 otherwise
316 * \see compute_memory_finalize_pending
317 */
compute_memory_promote_item(struct compute_memory_pool * pool,struct compute_memory_item * item,struct pipe_context * pipe,int64_t start_in_dw)318 static int compute_memory_promote_item(struct compute_memory_pool *pool,
319 struct compute_memory_item *item, struct pipe_context *pipe,
320 int64_t start_in_dw)
321 {
322 struct pipe_screen *screen = (struct pipe_screen *)pool->screen;
323 struct r600_context *rctx = (struct r600_context *)pipe;
324 struct pipe_resource *src = (struct pipe_resource *)item->real_buffer;
325 struct pipe_resource *dst = (struct pipe_resource *)pool->bo;
326 struct pipe_box box;
327
328 COMPUTE_DBG(pool->screen, "* compute_memory_promote_item()\n"
329 " + Promoting Item: %"PRIi64" , starting at: %"PRIi64" (%"PRIi64" bytes) "
330 "size: %"PRIi64" (%"PRIi64" bytes)\n\t\t\tnew start: %"PRIi64" (%"PRIi64" bytes)\n",
331 item->id, item->start_in_dw, item->start_in_dw * 4,
332 item->size_in_dw, item->size_in_dw * 4,
333 start_in_dw, start_in_dw * 4);
334
335 /* Remove the item from the unallocated list */
336 list_del(&item->link);
337
338 /* Add it back to the item_list */
339 list_addtail(&item->link, pool->item_list);
340 item->start_in_dw = start_in_dw;
341
342 if (src) {
343 u_box_1d(0, item->size_in_dw * 4, &box);
344
345 rctx->b.b.resource_copy_region(pipe,
346 dst, 0, item->start_in_dw * 4, 0 ,0,
347 src, 0, &box);
348
349 /* We check if the item is mapped for reading.
350 * In this case, we need to keep the temporary buffer 'alive'
351 * because it is possible to keep a map active for reading
352 * while a kernel (that reads from it) executes */
353 if (!(item->status & ITEM_MAPPED_FOR_READING)) {
354 pool->screen->b.b.resource_destroy(screen, src);
355 item->real_buffer = NULL;
356 }
357 }
358
359 return 0;
360 }
361
362 /**
363 * Moves an item from the \a item_list to the \a unallocated_list.
364 * \param item The item that will be demoted
365 * \see r600_compute_global_transfer_map
366 */
compute_memory_demote_item(struct compute_memory_pool * pool,struct compute_memory_item * item,struct pipe_context * pipe)367 void compute_memory_demote_item(struct compute_memory_pool *pool,
368 struct compute_memory_item *item, struct pipe_context *pipe)
369 {
370 struct r600_context *rctx = (struct r600_context *)pipe;
371 struct pipe_resource *src = (struct pipe_resource *)pool->bo;
372 struct pipe_resource *dst;
373 struct pipe_box box;
374
375 COMPUTE_DBG(pool->screen, "* compute_memory_demote_item()\n"
376 " + Demoting Item: %"PRIi64", starting at: %"PRIi64" (%"PRIi64" bytes) "
377 "size: %"PRIi64" (%"PRIi64" bytes)\n", item->id, item->start_in_dw,
378 item->start_in_dw * 4, item->size_in_dw, item->size_in_dw * 4);
379
380 /* First, we remove the item from the item_list */
381 list_del(&item->link);
382
383 /* Now we add it to the unallocated list */
384 list_addtail(&item->link, pool->unallocated_list);
385
386 /* We check if the intermediate buffer exists, and if it
387 * doesn't, we create it again */
388 if (item->real_buffer == NULL) {
389 item->real_buffer = r600_compute_buffer_alloc_vram(
390 pool->screen, item->size_in_dw * 4);
391 }
392
393 dst = (struct pipe_resource *)item->real_buffer;
394
395 /* We transfer the memory from the item in the pool to the
396 * temporary buffer */
397 u_box_1d(item->start_in_dw * 4, item->size_in_dw * 4, &box);
398
399 rctx->b.b.resource_copy_region(pipe,
400 dst, 0, 0, 0, 0,
401 src, 0, &box);
402
403 /* Remember to mark the buffer as 'pending' by setting start_in_dw to -1 */
404 item->start_in_dw = -1;
405
406 if (item->link.next != pool->item_list) {
407 pool->status |= POOL_FRAGMENTED;
408 }
409 }
410
411 /**
412 * Moves the item \a item forward from the resource \a src to the
413 * resource \a dst at \a new_start_in_dw
414 *
415 * This function assumes two things:
416 * 1) The item is \b only moved forward, unless src is different from dst
417 * 2) The item \b won't change it's position inside the \a item_list
418 *
419 * \param item The item that will be moved
420 * \param new_start_in_dw The new position of the item in \a item_list
421 * \see compute_memory_defrag
422 */
compute_memory_move_item(struct compute_memory_pool * pool,struct pipe_resource * src,struct pipe_resource * dst,struct compute_memory_item * item,uint64_t new_start_in_dw,struct pipe_context * pipe)423 static void compute_memory_move_item(struct compute_memory_pool *pool,
424 struct pipe_resource *src, struct pipe_resource *dst,
425 struct compute_memory_item *item, uint64_t new_start_in_dw,
426 struct pipe_context *pipe)
427 {
428 struct pipe_screen *screen = (struct pipe_screen *)pool->screen;
429 struct r600_context *rctx = (struct r600_context *)pipe;
430 struct pipe_box box;
431
432 COMPUTE_DBG(pool->screen, "* compute_memory_move_item()\n"
433 " + Moving item %"PRIi64" from %"PRIi64" (%"PRIi64" bytes) to %"PRIu64" (%"PRIu64" bytes)\n",
434 item->id, item->start_in_dw, item->start_in_dw * 4,
435 new_start_in_dw, new_start_in_dw * 4);
436
437 if (pool->item_list != item->link.prev) {
438 ASSERTED struct compute_memory_item *prev;
439 prev = container_of(item->link.prev, item, link);
440 assert(prev->start_in_dw + prev->size_in_dw <= new_start_in_dw);
441 }
442
443 u_box_1d(item->start_in_dw * 4, item->size_in_dw * 4, &box);
444
445 /* If the ranges don't overlap, or we are copying from one resource
446 * to another, we can just copy the item directly */
447 if (src != dst || new_start_in_dw + item->size_in_dw <= item->start_in_dw) {
448
449 rctx->b.b.resource_copy_region(pipe,
450 dst, 0, new_start_in_dw * 4, 0, 0,
451 src, 0, &box);
452 } else {
453 /* The ranges overlap, we will try first to use an intermediate
454 * resource to move the item */
455 struct pipe_resource *tmp = (struct pipe_resource *)
456 r600_compute_buffer_alloc_vram(pool->screen, item->size_in_dw * 4);
457
458 if (tmp != NULL) {
459 rctx->b.b.resource_copy_region(pipe,
460 tmp, 0, 0, 0, 0,
461 src, 0, &box);
462
463 box.x = 0;
464
465 rctx->b.b.resource_copy_region(pipe,
466 dst, 0, new_start_in_dw * 4, 0, 0,
467 tmp, 0, &box);
468
469 pool->screen->b.b.resource_destroy(screen, tmp);
470
471 } else {
472 /* The allocation of the temporary resource failed,
473 * falling back to use mappings */
474 uint32_t *map;
475 int64_t offset;
476 struct pipe_transfer *trans;
477
478 offset = item->start_in_dw - new_start_in_dw;
479
480 u_box_1d(new_start_in_dw * 4, (offset + item->size_in_dw) * 4, &box);
481
482 map = pipe->transfer_map(pipe, src, 0, PIPE_MAP_READ_WRITE,
483 &box, &trans);
484
485 assert(map);
486 assert(trans);
487
488 memmove(map, map + offset, item->size_in_dw * 4);
489
490 pipe->transfer_unmap(pipe, trans);
491 }
492 }
493
494 item->start_in_dw = new_start_in_dw;
495 }
496
497 /**
498 * Frees the memory asociated to the item with id \a id from the pool.
499 * \param id The id of the item to be freed.
500 */
compute_memory_free(struct compute_memory_pool * pool,int64_t id)501 void compute_memory_free(struct compute_memory_pool* pool, int64_t id)
502 {
503 struct compute_memory_item *item, *next;
504 struct pipe_screen *screen = (struct pipe_screen *)pool->screen;
505 struct pipe_resource *res;
506
507 COMPUTE_DBG(pool->screen, "* compute_memory_free() id + %"PRIi64" \n", id);
508
509 LIST_FOR_EACH_ENTRY_SAFE(item, next, pool->item_list, link) {
510
511 if (item->id == id) {
512
513 if (item->link.next != pool->item_list) {
514 pool->status |= POOL_FRAGMENTED;
515 }
516
517 list_del(&item->link);
518
519 if (item->real_buffer) {
520 res = (struct pipe_resource *)item->real_buffer;
521 pool->screen->b.b.resource_destroy(
522 screen, res);
523 }
524
525 free(item);
526
527 return;
528 }
529 }
530
531 LIST_FOR_EACH_ENTRY_SAFE(item, next, pool->unallocated_list, link) {
532
533 if (item->id == id) {
534 list_del(&item->link);
535
536 if (item->real_buffer) {
537 res = (struct pipe_resource *)item->real_buffer;
538 pool->screen->b.b.resource_destroy(
539 screen, res);
540 }
541
542 free(item);
543
544 return;
545 }
546 }
547
548 fprintf(stderr, "Internal error, invalid id %"PRIi64" "
549 "for compute_memory_free\n", id);
550
551 assert(0 && "error");
552 }
553
554 /**
555 * Creates pending allocations for new items, these items are
556 * placed in the unallocated_list.
557 * \param size_in_dw The size, in double words, of the new item.
558 * \return The new item
559 * \see r600_compute_global_buffer_create
560 */
compute_memory_alloc(struct compute_memory_pool * pool,int64_t size_in_dw)561 struct compute_memory_item* compute_memory_alloc(
562 struct compute_memory_pool* pool,
563 int64_t size_in_dw)
564 {
565 struct compute_memory_item *new_item = NULL;
566
567 COMPUTE_DBG(pool->screen, "* compute_memory_alloc() size_in_dw = %"PRIi64" (%"PRIi64" bytes)\n",
568 size_in_dw, 4 * size_in_dw);
569
570 new_item = (struct compute_memory_item *)
571 CALLOC(sizeof(struct compute_memory_item), 1);
572 if (!new_item)
573 return NULL;
574
575 new_item->size_in_dw = size_in_dw;
576 new_item->start_in_dw = -1; /* mark pending */
577 new_item->id = pool->next_id++;
578 new_item->pool = pool;
579 new_item->real_buffer = NULL;
580
581 list_addtail(&new_item->link, pool->unallocated_list);
582
583 COMPUTE_DBG(pool->screen, " + Adding item %p id = %"PRIi64" size = %"PRIi64" (%"PRIi64" bytes)\n",
584 new_item, new_item->id, new_item->size_in_dw,
585 new_item->size_in_dw * 4);
586 return new_item;
587 }
588
589 /**
590 * Transfer data host<->device, offset and size is in bytes.
591 * \param device_to_host 1 for device->host, 0 for host->device.
592 * \see compute_memory_shadow
593 */
compute_memory_transfer(struct compute_memory_pool * pool,struct pipe_context * pipe,int device_to_host,struct compute_memory_item * chunk,void * data,int offset_in_chunk,int size)594 static void compute_memory_transfer(
595 struct compute_memory_pool* pool,
596 struct pipe_context * pipe,
597 int device_to_host,
598 struct compute_memory_item* chunk,
599 void* data,
600 int offset_in_chunk,
601 int size)
602 {
603 int64_t aligned_size = pool->size_in_dw;
604 struct pipe_resource* gart = (struct pipe_resource*)pool->bo;
605 int64_t internal_offset = chunk->start_in_dw*4 + offset_in_chunk;
606
607 struct pipe_transfer *xfer;
608 uint32_t *map;
609
610 assert(gart);
611
612 COMPUTE_DBG(pool->screen, "* compute_memory_transfer() device_to_host = %d, "
613 "offset_in_chunk = %d, size = %d\n", device_to_host,
614 offset_in_chunk, size);
615
616 if (device_to_host) {
617 map = pipe->transfer_map(pipe, gart, 0, PIPE_MAP_READ,
618 &(struct pipe_box) { .width = aligned_size * 4,
619 .height = 1, .depth = 1 }, &xfer);
620 assert(xfer);
621 assert(map);
622 memcpy(data, map + internal_offset, size);
623 pipe->transfer_unmap(pipe, xfer);
624 } else {
625 map = pipe->transfer_map(pipe, gart, 0, PIPE_MAP_WRITE,
626 &(struct pipe_box) { .width = aligned_size * 4,
627 .height = 1, .depth = 1 }, &xfer);
628 assert(xfer);
629 assert(map);
630 memcpy(map + internal_offset, data, size);
631 pipe->transfer_unmap(pipe, xfer);
632 }
633 }
634