1 /*
2 * Copyright 2013 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Marek Olšák
25 */
26
27 #include "r600_cs.h"
28 #include "util/u_memory.h"
29 #include "util/u_upload_mgr.h"
30 #include <inttypes.h>
31 #include <stdio.h>
32
r600_rings_is_buffer_referenced(struct r600_common_context * ctx,struct pb_buffer * buf,enum radeon_bo_usage usage)33 bool r600_rings_is_buffer_referenced(struct r600_common_context *ctx,
34 struct pb_buffer *buf,
35 enum radeon_bo_usage usage)
36 {
37 if (ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs, buf, usage)) {
38 return true;
39 }
40 if (radeon_emitted(ctx->dma.cs, 0) &&
41 ctx->ws->cs_is_buffer_referenced(ctx->dma.cs, buf, usage)) {
42 return true;
43 }
44 return false;
45 }
46
r600_buffer_map_sync_with_rings(struct r600_common_context * ctx,struct r600_resource * resource,unsigned usage)47 void *r600_buffer_map_sync_with_rings(struct r600_common_context *ctx,
48 struct r600_resource *resource,
49 unsigned usage)
50 {
51 enum radeon_bo_usage rusage = RADEON_USAGE_READWRITE;
52 bool busy = false;
53
54 assert(!(resource->flags & RADEON_FLAG_SPARSE));
55
56 if (usage & PIPE_MAP_UNSYNCHRONIZED) {
57 return ctx->ws->buffer_map(resource->buf, NULL, usage);
58 }
59
60 if (!(usage & PIPE_MAP_WRITE)) {
61 /* have to wait for the last write */
62 rusage = RADEON_USAGE_WRITE;
63 }
64
65 if (radeon_emitted(ctx->gfx.cs, ctx->initial_gfx_cs_size) &&
66 ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs,
67 resource->buf, rusage)) {
68 if (usage & PIPE_MAP_DONTBLOCK) {
69 ctx->gfx.flush(ctx, PIPE_FLUSH_ASYNC, NULL);
70 return NULL;
71 } else {
72 ctx->gfx.flush(ctx, 0, NULL);
73 busy = true;
74 }
75 }
76 if (radeon_emitted(ctx->dma.cs, 0) &&
77 ctx->ws->cs_is_buffer_referenced(ctx->dma.cs,
78 resource->buf, rusage)) {
79 if (usage & PIPE_MAP_DONTBLOCK) {
80 ctx->dma.flush(ctx, PIPE_FLUSH_ASYNC, NULL);
81 return NULL;
82 } else {
83 ctx->dma.flush(ctx, 0, NULL);
84 busy = true;
85 }
86 }
87
88 if (busy || !ctx->ws->buffer_wait(resource->buf, 0, rusage)) {
89 if (usage & PIPE_MAP_DONTBLOCK) {
90 return NULL;
91 } else {
92 /* We will be wait for the GPU. Wait for any offloaded
93 * CS flush to complete to avoid busy-waiting in the winsys. */
94 ctx->ws->cs_sync_flush(ctx->gfx.cs);
95 if (ctx->dma.cs)
96 ctx->ws->cs_sync_flush(ctx->dma.cs);
97 }
98 }
99
100 /* Setting the CS to NULL will prevent doing checks we have done already. */
101 return ctx->ws->buffer_map(resource->buf, NULL, usage);
102 }
103
r600_init_resource_fields(struct r600_common_screen * rscreen,struct r600_resource * res,uint64_t size,unsigned alignment)104 void r600_init_resource_fields(struct r600_common_screen *rscreen,
105 struct r600_resource *res,
106 uint64_t size, unsigned alignment)
107 {
108 struct r600_texture *rtex = (struct r600_texture*)res;
109
110 res->bo_size = size;
111 res->bo_alignment = alignment;
112 res->flags = 0;
113 res->texture_handle_allocated = false;
114 res->image_handle_allocated = false;
115
116 switch (res->b.b.usage) {
117 case PIPE_USAGE_STREAM:
118 res->flags = RADEON_FLAG_GTT_WC;
119 /* fall through */
120 case PIPE_USAGE_STAGING:
121 /* Transfers are likely to occur more often with these
122 * resources. */
123 res->domains = RADEON_DOMAIN_GTT;
124 break;
125 case PIPE_USAGE_DYNAMIC:
126 /* Older kernels didn't always flush the HDP cache before
127 * CS execution
128 */
129 if (rscreen->info.drm_minor < 40) {
130 res->domains = RADEON_DOMAIN_GTT;
131 res->flags |= RADEON_FLAG_GTT_WC;
132 break;
133 }
134 /* fall through */
135 case PIPE_USAGE_DEFAULT:
136 case PIPE_USAGE_IMMUTABLE:
137 default:
138 /* Not listing GTT here improves performance in some
139 * apps. */
140 res->domains = RADEON_DOMAIN_VRAM;
141 res->flags |= RADEON_FLAG_GTT_WC;
142 break;
143 }
144
145 if (res->b.b.target == PIPE_BUFFER &&
146 res->b.b.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT |
147 PIPE_RESOURCE_FLAG_MAP_COHERENT)) {
148 /* Use GTT for all persistent mappings with older
149 * kernels, because they didn't always flush the HDP
150 * cache before CS execution.
151 *
152 * Write-combined CPU mappings are fine, the kernel
153 * ensures all CPU writes finish before the GPU
154 * executes a command stream.
155 */
156 if (rscreen->info.drm_minor < 40)
157 res->domains = RADEON_DOMAIN_GTT;
158 }
159
160 /* Tiled textures are unmappable. Always put them in VRAM. */
161 if ((res->b.b.target != PIPE_BUFFER && !rtex->surface.is_linear) ||
162 res->flags & R600_RESOURCE_FLAG_UNMAPPABLE) {
163 res->domains = RADEON_DOMAIN_VRAM;
164 res->flags |= RADEON_FLAG_NO_CPU_ACCESS |
165 RADEON_FLAG_GTT_WC;
166 }
167
168 /* Displayable and shareable surfaces are not suballocated. */
169 if (res->b.b.bind & (PIPE_BIND_SHARED | PIPE_BIND_SCANOUT))
170 res->flags |= RADEON_FLAG_NO_SUBALLOC; /* shareable */
171 else
172 res->flags |= RADEON_FLAG_NO_INTERPROCESS_SHARING;
173
174 if (rscreen->debug_flags & DBG_NO_WC)
175 res->flags &= ~RADEON_FLAG_GTT_WC;
176
177 /* Set expected VRAM and GART usage for the buffer. */
178 res->vram_usage = 0;
179 res->gart_usage = 0;
180
181 if (res->domains & RADEON_DOMAIN_VRAM)
182 res->vram_usage = size;
183 else if (res->domains & RADEON_DOMAIN_GTT)
184 res->gart_usage = size;
185 }
186
r600_alloc_resource(struct r600_common_screen * rscreen,struct r600_resource * res)187 bool r600_alloc_resource(struct r600_common_screen *rscreen,
188 struct r600_resource *res)
189 {
190 struct pb_buffer *old_buf, *new_buf;
191
192 /* Allocate a new resource. */
193 new_buf = rscreen->ws->buffer_create(rscreen->ws, res->bo_size,
194 res->bo_alignment,
195 res->domains, res->flags);
196 if (!new_buf) {
197 return false;
198 }
199
200 /* Replace the pointer such that if res->buf wasn't NULL, it won't be
201 * NULL. This should prevent crashes with multiple contexts using
202 * the same buffer where one of the contexts invalidates it while
203 * the others are using it. */
204 old_buf = res->buf;
205 res->buf = new_buf; /* should be atomic */
206
207 if (rscreen->info.r600_has_virtual_memory)
208 res->gpu_address = rscreen->ws->buffer_get_virtual_address(res->buf);
209 else
210 res->gpu_address = 0;
211
212 pb_reference(&old_buf, NULL);
213
214 util_range_set_empty(&res->valid_buffer_range);
215
216 /* Print debug information. */
217 if (rscreen->debug_flags & DBG_VM && res->b.b.target == PIPE_BUFFER) {
218 fprintf(stderr, "VM start=0x%"PRIX64" end=0x%"PRIX64" | Buffer %"PRIu64" bytes\n",
219 res->gpu_address, res->gpu_address + res->buf->size,
220 res->buf->size);
221 }
222 return true;
223 }
224
r600_buffer_destroy(struct pipe_screen * screen,struct pipe_resource * buf)225 static void r600_buffer_destroy(struct pipe_screen *screen,
226 struct pipe_resource *buf)
227 {
228 struct r600_resource *rbuffer = r600_resource(buf);
229
230 threaded_resource_deinit(buf);
231 util_range_destroy(&rbuffer->valid_buffer_range);
232 pipe_resource_reference((struct pipe_resource**)&rbuffer->immed_buffer, NULL);
233 pb_reference(&rbuffer->buf, NULL);
234 FREE(rbuffer);
235 }
236
237 static bool
r600_invalidate_buffer(struct r600_common_context * rctx,struct r600_resource * rbuffer)238 r600_invalidate_buffer(struct r600_common_context *rctx,
239 struct r600_resource *rbuffer)
240 {
241 /* Shared buffers can't be reallocated. */
242 if (rbuffer->b.is_shared)
243 return false;
244
245 /* Sparse buffers can't be reallocated. */
246 if (rbuffer->flags & RADEON_FLAG_SPARSE)
247 return false;
248
249 /* In AMD_pinned_memory, the user pointer association only gets
250 * broken when the buffer is explicitly re-allocated.
251 */
252 if (rbuffer->b.is_user_ptr)
253 return false;
254
255 /* Check if mapping this buffer would cause waiting for the GPU. */
256 if (r600_rings_is_buffer_referenced(rctx, rbuffer->buf, RADEON_USAGE_READWRITE) ||
257 !rctx->ws->buffer_wait(rbuffer->buf, 0, RADEON_USAGE_READWRITE)) {
258 rctx->invalidate_buffer(&rctx->b, &rbuffer->b.b);
259 } else {
260 util_range_set_empty(&rbuffer->valid_buffer_range);
261 }
262
263 return true;
264 }
265
266 /* Replace the storage of dst with src. */
r600_replace_buffer_storage(struct pipe_context * ctx,struct pipe_resource * dst,struct pipe_resource * src)267 void r600_replace_buffer_storage(struct pipe_context *ctx,
268 struct pipe_resource *dst,
269 struct pipe_resource *src)
270 {
271 struct r600_common_context *rctx = (struct r600_common_context *)ctx;
272 struct r600_resource *rdst = r600_resource(dst);
273 struct r600_resource *rsrc = r600_resource(src);
274 uint64_t old_gpu_address = rdst->gpu_address;
275
276 pb_reference(&rdst->buf, rsrc->buf);
277 rdst->gpu_address = rsrc->gpu_address;
278 rdst->b.b.bind = rsrc->b.b.bind;
279 rdst->flags = rsrc->flags;
280
281 assert(rdst->vram_usage == rsrc->vram_usage);
282 assert(rdst->gart_usage == rsrc->gart_usage);
283 assert(rdst->bo_size == rsrc->bo_size);
284 assert(rdst->bo_alignment == rsrc->bo_alignment);
285 assert(rdst->domains == rsrc->domains);
286
287 rctx->rebind_buffer(ctx, dst, old_gpu_address);
288 }
289
r600_invalidate_resource(struct pipe_context * ctx,struct pipe_resource * resource)290 void r600_invalidate_resource(struct pipe_context *ctx,
291 struct pipe_resource *resource)
292 {
293 struct r600_common_context *rctx = (struct r600_common_context*)ctx;
294 struct r600_resource *rbuffer = r600_resource(resource);
295
296 /* We currently only do anyting here for buffers */
297 if (resource->target == PIPE_BUFFER)
298 (void)r600_invalidate_buffer(rctx, rbuffer);
299 }
300
r600_buffer_get_transfer(struct pipe_context * ctx,struct pipe_resource * resource,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** ptransfer,void * data,struct r600_resource * staging,unsigned offset)301 static void *r600_buffer_get_transfer(struct pipe_context *ctx,
302 struct pipe_resource *resource,
303 unsigned usage,
304 const struct pipe_box *box,
305 struct pipe_transfer **ptransfer,
306 void *data, struct r600_resource *staging,
307 unsigned offset)
308 {
309 struct r600_common_context *rctx = (struct r600_common_context*)ctx;
310 struct r600_transfer *transfer;
311
312 if (usage & TC_TRANSFER_MAP_THREADED_UNSYNC)
313 transfer = slab_alloc(&rctx->pool_transfers_unsync);
314 else
315 transfer = slab_alloc(&rctx->pool_transfers);
316
317 transfer->b.b.resource = NULL;
318 pipe_resource_reference(&transfer->b.b.resource, resource);
319 transfer->b.b.level = 0;
320 transfer->b.b.usage = usage;
321 transfer->b.b.box = *box;
322 transfer->b.b.stride = 0;
323 transfer->b.b.layer_stride = 0;
324 transfer->b.staging = NULL;
325 transfer->offset = offset;
326 transfer->staging = staging;
327 *ptransfer = &transfer->b.b;
328 return data;
329 }
330
r600_can_dma_copy_buffer(struct r600_common_context * rctx,unsigned dstx,unsigned srcx,unsigned size)331 static bool r600_can_dma_copy_buffer(struct r600_common_context *rctx,
332 unsigned dstx, unsigned srcx, unsigned size)
333 {
334 bool dword_aligned = !(dstx % 4) && !(srcx % 4) && !(size % 4);
335
336 return rctx->screen->has_cp_dma ||
337 (dword_aligned && (rctx->dma.cs ||
338 rctx->screen->has_streamout));
339
340 }
341
r600_buffer_transfer_map(struct pipe_context * ctx,struct pipe_resource * resource,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** ptransfer)342 static void *r600_buffer_transfer_map(struct pipe_context *ctx,
343 struct pipe_resource *resource,
344 unsigned level,
345 unsigned usage,
346 const struct pipe_box *box,
347 struct pipe_transfer **ptransfer)
348 {
349 struct r600_common_context *rctx = (struct r600_common_context*)ctx;
350 struct r600_common_screen *rscreen = (struct r600_common_screen*)ctx->screen;
351 struct r600_resource *rbuffer = r600_resource(resource);
352 uint8_t *data;
353
354 assert(box->x + box->width <= resource->width0);
355
356 /* From GL_AMD_pinned_memory issues:
357 *
358 * 4) Is glMapBuffer on a shared buffer guaranteed to return the
359 * same system address which was specified at creation time?
360 *
361 * RESOLVED: NO. The GL implementation might return a different
362 * virtual mapping of that memory, although the same physical
363 * page will be used.
364 *
365 * So don't ever use staging buffers.
366 */
367 if (rbuffer->b.is_user_ptr)
368 usage |= PIPE_MAP_PERSISTENT;
369
370 /* See if the buffer range being mapped has never been initialized,
371 * in which case it can be mapped unsynchronized. */
372 if (!(usage & (PIPE_MAP_UNSYNCHRONIZED |
373 TC_TRANSFER_MAP_NO_INFER_UNSYNCHRONIZED)) &&
374 usage & PIPE_MAP_WRITE &&
375 !rbuffer->b.is_shared &&
376 !util_ranges_intersect(&rbuffer->valid_buffer_range, box->x, box->x + box->width)) {
377 usage |= PIPE_MAP_UNSYNCHRONIZED;
378 }
379
380 /* If discarding the entire range, discard the whole resource instead. */
381 if (usage & PIPE_MAP_DISCARD_RANGE &&
382 box->x == 0 && box->width == resource->width0) {
383 usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
384 }
385
386 if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE &&
387 !(usage & (PIPE_MAP_UNSYNCHRONIZED |
388 TC_TRANSFER_MAP_NO_INVALIDATE))) {
389 assert(usage & PIPE_MAP_WRITE);
390
391 if (r600_invalidate_buffer(rctx, rbuffer)) {
392 /* At this point, the buffer is always idle. */
393 usage |= PIPE_MAP_UNSYNCHRONIZED;
394 } else {
395 /* Fall back to a temporary buffer. */
396 usage |= PIPE_MAP_DISCARD_RANGE;
397 }
398 }
399
400 if ((usage & PIPE_MAP_DISCARD_RANGE) &&
401 !(rscreen->debug_flags & DBG_NO_DISCARD_RANGE) &&
402 ((!(usage & (PIPE_MAP_UNSYNCHRONIZED |
403 PIPE_MAP_PERSISTENT)) &&
404 r600_can_dma_copy_buffer(rctx, box->x, 0, box->width)) ||
405 (rbuffer->flags & RADEON_FLAG_SPARSE))) {
406 assert(usage & PIPE_MAP_WRITE);
407
408 /* Check if mapping this buffer would cause waiting for the GPU.
409 */
410 if (rbuffer->flags & RADEON_FLAG_SPARSE ||
411 r600_rings_is_buffer_referenced(rctx, rbuffer->buf, RADEON_USAGE_READWRITE) ||
412 !rctx->ws->buffer_wait(rbuffer->buf, 0, RADEON_USAGE_READWRITE)) {
413 /* Do a wait-free write-only transfer using a temporary buffer. */
414 unsigned offset;
415 struct r600_resource *staging = NULL;
416
417 u_upload_alloc(ctx->stream_uploader, 0,
418 box->width + (box->x % R600_MAP_BUFFER_ALIGNMENT),
419 rctx->screen->info.tcc_cache_line_size,
420 &offset, (struct pipe_resource**)&staging,
421 (void**)&data);
422
423 if (staging) {
424 data += box->x % R600_MAP_BUFFER_ALIGNMENT;
425 return r600_buffer_get_transfer(ctx, resource, usage, box,
426 ptransfer, data, staging, offset);
427 } else if (rbuffer->flags & RADEON_FLAG_SPARSE) {
428 return NULL;
429 }
430 } else {
431 /* At this point, the buffer is always idle (we checked it above). */
432 usage |= PIPE_MAP_UNSYNCHRONIZED;
433 }
434 }
435 /* Use a staging buffer in cached GTT for reads. */
436 else if (((usage & PIPE_MAP_READ) &&
437 !(usage & PIPE_MAP_PERSISTENT) &&
438 (rbuffer->domains & RADEON_DOMAIN_VRAM ||
439 rbuffer->flags & RADEON_FLAG_GTT_WC) &&
440 r600_can_dma_copy_buffer(rctx, 0, box->x, box->width)) ||
441 (rbuffer->flags & RADEON_FLAG_SPARSE)) {
442 struct r600_resource *staging;
443
444 assert(!(usage & TC_TRANSFER_MAP_THREADED_UNSYNC));
445 staging = (struct r600_resource*) pipe_buffer_create(
446 ctx->screen, 0, PIPE_USAGE_STAGING,
447 box->width + (box->x % R600_MAP_BUFFER_ALIGNMENT));
448 if (staging) {
449 /* Copy the VRAM buffer to the staging buffer. */
450 rctx->dma_copy(ctx, &staging->b.b, 0,
451 box->x % R600_MAP_BUFFER_ALIGNMENT,
452 0, 0, resource, 0, box);
453
454 data = r600_buffer_map_sync_with_rings(rctx, staging,
455 usage & ~PIPE_MAP_UNSYNCHRONIZED);
456 if (!data) {
457 r600_resource_reference(&staging, NULL);
458 return NULL;
459 }
460 data += box->x % R600_MAP_BUFFER_ALIGNMENT;
461
462 return r600_buffer_get_transfer(ctx, resource, usage, box,
463 ptransfer, data, staging, 0);
464 } else if (rbuffer->flags & RADEON_FLAG_SPARSE) {
465 return NULL;
466 }
467 }
468
469 data = r600_buffer_map_sync_with_rings(rctx, rbuffer, usage);
470 if (!data) {
471 return NULL;
472 }
473 data += box->x;
474
475 return r600_buffer_get_transfer(ctx, resource, usage, box,
476 ptransfer, data, NULL, 0);
477 }
478
r600_buffer_do_flush_region(struct pipe_context * ctx,struct pipe_transfer * transfer,const struct pipe_box * box)479 static void r600_buffer_do_flush_region(struct pipe_context *ctx,
480 struct pipe_transfer *transfer,
481 const struct pipe_box *box)
482 {
483 struct r600_transfer *rtransfer = (struct r600_transfer*)transfer;
484 struct r600_resource *rbuffer = r600_resource(transfer->resource);
485
486 if (rtransfer->staging) {
487 struct pipe_resource *dst, *src;
488 unsigned soffset;
489 struct pipe_box dma_box;
490
491 dst = transfer->resource;
492 src = &rtransfer->staging->b.b;
493 soffset = rtransfer->offset + box->x % R600_MAP_BUFFER_ALIGNMENT;
494
495 u_box_1d(soffset, box->width, &dma_box);
496
497 /* Copy the staging buffer into the original one. */
498 ctx->resource_copy_region(ctx, dst, 0, box->x, 0, 0, src, 0, &dma_box);
499 }
500
501 util_range_add(&rbuffer->b.b, &rbuffer->valid_buffer_range, box->x,
502 box->x + box->width);
503 }
504
r600_buffer_flush_region(struct pipe_context * ctx,struct pipe_transfer * transfer,const struct pipe_box * rel_box)505 static void r600_buffer_flush_region(struct pipe_context *ctx,
506 struct pipe_transfer *transfer,
507 const struct pipe_box *rel_box)
508 {
509 unsigned required_usage = PIPE_MAP_WRITE |
510 PIPE_MAP_FLUSH_EXPLICIT;
511
512 if ((transfer->usage & required_usage) == required_usage) {
513 struct pipe_box box;
514
515 u_box_1d(transfer->box.x + rel_box->x, rel_box->width, &box);
516 r600_buffer_do_flush_region(ctx, transfer, &box);
517 }
518 }
519
r600_buffer_transfer_unmap(struct pipe_context * ctx,struct pipe_transfer * transfer)520 static void r600_buffer_transfer_unmap(struct pipe_context *ctx,
521 struct pipe_transfer *transfer)
522 {
523 struct r600_common_context *rctx = (struct r600_common_context*)ctx;
524 struct r600_transfer *rtransfer = (struct r600_transfer*)transfer;
525
526 if (transfer->usage & PIPE_MAP_WRITE &&
527 !(transfer->usage & PIPE_MAP_FLUSH_EXPLICIT))
528 r600_buffer_do_flush_region(ctx, transfer, &transfer->box);
529
530 r600_resource_reference(&rtransfer->staging, NULL);
531 assert(rtransfer->b.staging == NULL); /* for threaded context only */
532 pipe_resource_reference(&transfer->resource, NULL);
533
534 /* Don't use pool_transfers_unsync. We are always in the driver
535 * thread. */
536 slab_free(&rctx->pool_transfers, transfer);
537 }
538
r600_buffer_subdata(struct pipe_context * ctx,struct pipe_resource * buffer,unsigned usage,unsigned offset,unsigned size,const void * data)539 void r600_buffer_subdata(struct pipe_context *ctx,
540 struct pipe_resource *buffer,
541 unsigned usage, unsigned offset,
542 unsigned size, const void *data)
543 {
544 struct pipe_transfer *transfer = NULL;
545 struct pipe_box box;
546 uint8_t *map = NULL;
547
548 usage |= PIPE_MAP_WRITE;
549
550 if (!(usage & PIPE_MAP_DIRECTLY))
551 usage |= PIPE_MAP_DISCARD_RANGE;
552
553 u_box_1d(offset, size, &box);
554 map = r600_buffer_transfer_map(ctx, buffer, 0, usage, &box, &transfer);
555 if (!map)
556 return;
557
558 memcpy(map, data, size);
559 r600_buffer_transfer_unmap(ctx, transfer);
560 }
561
562 static const struct u_resource_vtbl r600_buffer_vtbl =
563 {
564 NULL, /* get_handle */
565 r600_buffer_destroy, /* resource_destroy */
566 r600_buffer_transfer_map, /* transfer_map */
567 r600_buffer_flush_region, /* transfer_flush_region */
568 r600_buffer_transfer_unmap, /* transfer_unmap */
569 };
570
571 static struct r600_resource *
r600_alloc_buffer_struct(struct pipe_screen * screen,const struct pipe_resource * templ)572 r600_alloc_buffer_struct(struct pipe_screen *screen,
573 const struct pipe_resource *templ)
574 {
575 struct r600_resource *rbuffer;
576
577 rbuffer = MALLOC_STRUCT(r600_resource);
578
579 rbuffer->b.b = *templ;
580 rbuffer->b.b.next = NULL;
581 pipe_reference_init(&rbuffer->b.b.reference, 1);
582 rbuffer->b.b.screen = screen;
583
584 rbuffer->b.vtbl = &r600_buffer_vtbl;
585 threaded_resource_init(&rbuffer->b.b);
586
587 rbuffer->buf = NULL;
588 rbuffer->bind_history = 0;
589 rbuffer->immed_buffer = NULL;
590 util_range_init(&rbuffer->valid_buffer_range);
591 return rbuffer;
592 }
593
r600_buffer_create(struct pipe_screen * screen,const struct pipe_resource * templ,unsigned alignment)594 struct pipe_resource *r600_buffer_create(struct pipe_screen *screen,
595 const struct pipe_resource *templ,
596 unsigned alignment)
597 {
598 struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
599 struct r600_resource *rbuffer = r600_alloc_buffer_struct(screen, templ);
600
601 r600_init_resource_fields(rscreen, rbuffer, templ->width0, alignment);
602
603 if (templ->flags & PIPE_RESOURCE_FLAG_SPARSE)
604 rbuffer->flags |= RADEON_FLAG_SPARSE;
605
606 if (!r600_alloc_resource(rscreen, rbuffer)) {
607 FREE(rbuffer);
608 return NULL;
609 }
610 return &rbuffer->b.b;
611 }
612
r600_aligned_buffer_create(struct pipe_screen * screen,unsigned flags,unsigned usage,unsigned size,unsigned alignment)613 struct pipe_resource *r600_aligned_buffer_create(struct pipe_screen *screen,
614 unsigned flags,
615 unsigned usage,
616 unsigned size,
617 unsigned alignment)
618 {
619 struct pipe_resource buffer;
620
621 memset(&buffer, 0, sizeof buffer);
622 buffer.target = PIPE_BUFFER;
623 buffer.format = PIPE_FORMAT_R8_UNORM;
624 buffer.bind = 0;
625 buffer.usage = usage;
626 buffer.flags = flags;
627 buffer.width0 = size;
628 buffer.height0 = 1;
629 buffer.depth0 = 1;
630 buffer.array_size = 1;
631 return r600_buffer_create(screen, &buffer, alignment);
632 }
633
634 struct pipe_resource *
r600_buffer_from_user_memory(struct pipe_screen * screen,const struct pipe_resource * templ,void * user_memory)635 r600_buffer_from_user_memory(struct pipe_screen *screen,
636 const struct pipe_resource *templ,
637 void *user_memory)
638 {
639 struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
640 struct radeon_winsys *ws = rscreen->ws;
641 struct r600_resource *rbuffer = r600_alloc_buffer_struct(screen, templ);
642
643 rbuffer->domains = RADEON_DOMAIN_GTT;
644 rbuffer->flags = 0;
645 rbuffer->b.is_user_ptr = true;
646 util_range_add(&rbuffer->b.b, &rbuffer->valid_buffer_range, 0, templ->width0);
647 util_range_add(&rbuffer->b.b, &rbuffer->b.valid_buffer_range, 0, templ->width0);
648
649 /* Convert a user pointer to a buffer. */
650 rbuffer->buf = ws->buffer_from_ptr(ws, user_memory, templ->width0);
651 if (!rbuffer->buf) {
652 FREE(rbuffer);
653 return NULL;
654 }
655
656 if (rscreen->info.r600_has_virtual_memory)
657 rbuffer->gpu_address =
658 ws->buffer_get_virtual_address(rbuffer->buf);
659 else
660 rbuffer->gpu_address = 0;
661
662 rbuffer->vram_usage = 0;
663 rbuffer->gart_usage = templ->width0;
664
665 return &rbuffer->b.b;
666 }
667