1 /**************************************************************************
2 *
3 * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 /* Originally a fake version of the buffer manager so that we can
29 * prototype the changes in a driver fairly quickly, has been fleshed
30 * out to a fully functional interim solution.
31 *
32 * Basically wraps the old style memory management in the new
33 * programming interface, but is more expressive and avoids many of
34 * the bugs in the old texture manager.
35 */
36
37 #ifdef HAVE_CONFIG_H
38 #include "config.h"
39 #endif
40
41 #include <stdlib.h>
42 #include <string.h>
43 #include <assert.h>
44 #include <errno.h>
45 #include <xf86drm.h>
46 #include <pthread.h>
47 #include "intel_bufmgr.h"
48 #include "intel_bufmgr_priv.h"
49 #include "drm.h"
50 #include "i915_drm.h"
51 #include "mm.h"
52 #include "libdrm_lists.h"
53
54 #define ALIGN(value, alignment) ((value + alignment - 1) & ~(alignment - 1))
55
56 #define DBG(...) do { \
57 if (bufmgr_fake->bufmgr.debug) \
58 drmMsg(__VA_ARGS__); \
59 } while (0)
60
61 /* Internal flags:
62 */
63 #define BM_NO_BACKING_STORE 0x00000001
64 #define BM_NO_FENCE_SUBDATA 0x00000002
65 #define BM_PINNED 0x00000004
66
67 /* Wrapper around mm.c's mem_block, which understands that you must
68 * wait for fences to expire before memory can be freed. This is
69 * specific to our use of memcpy for uploads - an upload that was
70 * processed through the command queue wouldn't need to care about
71 * fences.
72 */
73 #define MAX_RELOCS 4096
74
75 struct fake_buffer_reloc
76 {
77 /** Buffer object that the relocation points at. */
78 drm_intel_bo *target_buf;
79 /** Offset of the relocation entry within reloc_buf. */
80 uint32_t offset;
81 /** Cached value of the offset when we last performed this relocation. */
82 uint32_t last_target_offset;
83 /** Value added to target_buf's offset to get the relocation entry. */
84 uint32_t delta;
85 /** Cache domains the target buffer is read into. */
86 uint32_t read_domains;
87 /** Cache domain the target buffer will have dirty cachelines in. */
88 uint32_t write_domain;
89 };
90
91 struct block {
92 struct block *next, *prev;
93 struct mem_block *mem; /* BM_MEM_AGP */
94
95 /**
96 * Marks that the block is currently in the aperture and has yet to be
97 * fenced.
98 */
99 unsigned on_hardware:1;
100 /**
101 * Marks that the block is currently fenced (being used by rendering) and
102 * can't be freed until @fence is passed.
103 */
104 unsigned fenced:1;
105
106 /** Fence cookie for the block. */
107 unsigned fence; /* Split to read_fence, write_fence */
108
109 drm_intel_bo *bo;
110 void *virtual;
111 };
112
113 typedef struct _bufmgr_fake {
114 drm_intel_bufmgr bufmgr;
115
116 pthread_mutex_t lock;
117
118 unsigned long low_offset;
119 unsigned long size;
120 void *virtual;
121
122 struct mem_block *heap;
123
124 unsigned buf_nr; /* for generating ids */
125
126 /**
127 * List of blocks which are currently in the GART but haven't been
128 * fenced yet.
129 */
130 struct block on_hardware;
131 /**
132 * List of blocks which are in the GART and have an active fence on them.
133 */
134 struct block fenced;
135 /**
136 * List of blocks which have an expired fence and are ready to be evicted.
137 */
138 struct block lru;
139
140 unsigned int last_fence;
141
142 unsigned fail:1;
143 unsigned need_fence:1;
144 int thrashing;
145
146 /**
147 * Driver callback to emit a fence, returning the cookie.
148 *
149 * This allows the driver to hook in a replacement for the DRM usage in
150 * bufmgr_fake.
151 *
152 * Currently, this also requires that a write flush be emitted before
153 * emitting the fence, but this should change.
154 */
155 unsigned int (*fence_emit)(void *private);
156 /** Driver callback to wait for a fence cookie to have passed. */
157 void (*fence_wait)(unsigned int fence, void *private);
158 void *fence_priv;
159
160 /**
161 * Driver callback to execute a buffer.
162 *
163 * This allows the driver to hook in a replacement for the DRM usage in
164 * bufmgr_fake.
165 */
166 int (*exec)(drm_intel_bo *bo, unsigned int used, void *priv);
167 void *exec_priv;
168
169 /** Driver-supplied argument to driver callbacks */
170 void *driver_priv;
171 /* Pointer to kernel-updated sarea data for the last completed user irq */
172 volatile int *last_dispatch;
173
174 int fd;
175
176 int debug;
177
178 int performed_rendering;
179 } drm_intel_bufmgr_fake;
180
181 typedef struct _drm_intel_bo_fake {
182 drm_intel_bo bo;
183
184 unsigned id; /* debug only */
185 const char *name;
186
187 unsigned dirty:1;
188 /** has the card written to this buffer - we make need to copy it back */
189 unsigned card_dirty:1;
190 unsigned int refcount;
191 /* Flags may consist of any of the DRM_BO flags, plus
192 * DRM_BO_NO_BACKING_STORE and BM_NO_FENCE_SUBDATA, which are the first two
193 * driver private flags.
194 */
195 uint64_t flags;
196 /** Cache domains the target buffer is read into. */
197 uint32_t read_domains;
198 /** Cache domain the target buffer will have dirty cachelines in. */
199 uint32_t write_domain;
200
201 unsigned int alignment;
202 int is_static, validated;
203 unsigned int map_count;
204
205 /** relocation list */
206 struct fake_buffer_reloc *relocs;
207 int nr_relocs;
208 /**
209 * Total size of the target_bos of this buffer.
210 *
211 * Used for estimation in check_aperture.
212 */
213 unsigned int child_size;
214
215 struct block *block;
216 void *backing_store;
217 void (*invalidate_cb)(drm_intel_bo *bo, void *ptr);
218 void *invalidate_ptr;
219 } drm_intel_bo_fake;
220
221 static int clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake,
222 unsigned int fence_cookie);
223
224 #define MAXFENCE 0x7fffffff
225
FENCE_LTE(unsigned a,unsigned b)226 static int FENCE_LTE( unsigned a, unsigned b )
227 {
228 if (a == b)
229 return 1;
230
231 if (a < b && b - a < (1<<24))
232 return 1;
233
234 if (a > b && MAXFENCE - a + b < (1<<24))
235 return 1;
236
237 return 0;
238 }
239
drm_intel_bufmgr_fake_set_fence_callback(drm_intel_bufmgr * bufmgr,unsigned int (* emit)(void * priv),void (* wait)(unsigned int fence,void * priv),void * priv)240 void drm_intel_bufmgr_fake_set_fence_callback(drm_intel_bufmgr *bufmgr,
241 unsigned int (*emit)(void *priv),
242 void (*wait)(unsigned int fence,
243 void *priv),
244 void *priv)
245 {
246 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bufmgr;
247
248 bufmgr_fake->fence_emit = emit;
249 bufmgr_fake->fence_wait = wait;
250 bufmgr_fake->fence_priv = priv;
251 }
252
253 static unsigned int
_fence_emit_internal(drm_intel_bufmgr_fake * bufmgr_fake)254 _fence_emit_internal(drm_intel_bufmgr_fake *bufmgr_fake)
255 {
256 struct drm_i915_irq_emit ie;
257 int ret, seq = 1;
258
259 if (bufmgr_fake->fence_emit != NULL) {
260 seq = bufmgr_fake->fence_emit(bufmgr_fake->fence_priv);
261 return seq;
262 }
263
264 ie.irq_seq = &seq;
265 ret = drmCommandWriteRead(bufmgr_fake->fd, DRM_I915_IRQ_EMIT,
266 &ie, sizeof(ie));
267 if (ret) {
268 drmMsg("%s: drm_i915_irq_emit: %d\n", __FUNCTION__, ret);
269 abort();
270 }
271
272 DBG("emit 0x%08x\n", seq);
273 return seq;
274 }
275
276 static void
_fence_wait_internal(drm_intel_bufmgr_fake * bufmgr_fake,int seq)277 _fence_wait_internal(drm_intel_bufmgr_fake *bufmgr_fake, int seq)
278 {
279 struct drm_i915_irq_wait iw;
280 int hw_seq, busy_count = 0;
281 int ret;
282 int kernel_lied;
283
284 if (bufmgr_fake->fence_wait != NULL) {
285 bufmgr_fake->fence_wait(seq, bufmgr_fake->fence_priv);
286 clear_fenced(bufmgr_fake, seq);
287 return;
288 }
289
290 DBG("wait 0x%08x\n", iw.irq_seq);
291
292 iw.irq_seq = seq;
293
294 /* The kernel IRQ_WAIT implementation is all sorts of broken.
295 * 1) It returns 1 to 0x7fffffff instead of using the full 32-bit unsigned
296 * range.
297 * 2) It returns 0 if hw_seq >= seq, not seq - hw_seq < 0 on the 32-bit
298 * signed range.
299 * 3) It waits if seq < hw_seq, not seq - hw_seq > 0 on the 32-bit
300 * signed range.
301 * 4) It returns -EBUSY in 3 seconds even if the hardware is still
302 * successfully chewing through buffers.
303 *
304 * Assume that in userland we treat sequence numbers as ints, which makes
305 * some of the comparisons convenient, since the sequence numbers are
306 * all postive signed integers.
307 *
308 * From this we get several cases we need to handle. Here's a timeline.
309 * 0x2 0x7 0x7ffffff8 0x7ffffffd
310 * | | | |
311 * -------------------------------------------------------------------
312 *
313 * A) Normal wait for hw to catch up
314 * hw_seq seq
315 * | |
316 * -------------------------------------------------------------------
317 * seq - hw_seq = 5. If we call IRQ_WAIT, it will wait for hw to catch up.
318 *
319 * B) Normal wait for a sequence number that's already passed.
320 * seq hw_seq
321 * | |
322 * -------------------------------------------------------------------
323 * seq - hw_seq = -5. If we call IRQ_WAIT, it returns 0 quickly.
324 *
325 * C) Hardware has already wrapped around ahead of us
326 * hw_seq seq
327 * | |
328 * -------------------------------------------------------------------
329 * seq - hw_seq = 0x80000000 - 5. If we called IRQ_WAIT, it would wait
330 * for hw_seq >= seq, which may never occur. Thus, we want to catch this
331 * in userland and return 0.
332 *
333 * D) We've wrapped around ahead of the hardware.
334 * seq hw_seq
335 * | |
336 * -------------------------------------------------------------------
337 * seq - hw_seq = -(0x80000000 - 5). If we called IRQ_WAIT, it would return
338 * 0 quickly because hw_seq >= seq, even though the hardware isn't caught up.
339 * Thus, we need to catch this early return in userland and bother the
340 * kernel until the hardware really does catch up.
341 *
342 * E) Hardware might wrap after we test in userland.
343 * hw_seq seq
344 * | |
345 * -------------------------------------------------------------------
346 * seq - hw_seq = 5. If we call IRQ_WAIT, it will likely see seq >= hw_seq
347 * and wait. However, suppose hw_seq wraps before we make it into the
348 * kernel. The kernel sees hw_seq >= seq and waits for 3 seconds then
349 * returns -EBUSY. This is case C). We should catch this and then return
350 * successfully.
351 *
352 * F) Hardware might take a long time on a buffer.
353 * hw_seq seq
354 * | |
355 * -------------------------------------------------------------------
356 * seq - hw_seq = 5. If we call IRQ_WAIT, if sequence 2 through 5 take too
357 * long, it will return -EBUSY. Batchbuffers in the gltestperf demo were
358 * seen to take up to 7 seconds. We should catch early -EBUSY return
359 * and keep trying.
360 */
361
362 do {
363 /* Keep a copy of last_dispatch so that if the wait -EBUSYs because the
364 * hardware didn't catch up in 3 seconds, we can see if it at least made
365 * progress and retry.
366 */
367 hw_seq = *bufmgr_fake->last_dispatch;
368
369 /* Catch case C */
370 if (seq - hw_seq > 0x40000000)
371 return;
372
373 ret = drmCommandWrite(bufmgr_fake->fd, DRM_I915_IRQ_WAIT,
374 &iw, sizeof(iw));
375 /* Catch case D */
376 kernel_lied = (ret == 0) && (seq - *bufmgr_fake->last_dispatch <
377 -0x40000000);
378
379 /* Catch case E */
380 if (ret == -EBUSY && (seq - *bufmgr_fake->last_dispatch > 0x40000000))
381 ret = 0;
382
383 /* Catch case F: Allow up to 15 seconds chewing on one buffer. */
384 if ((ret == -EBUSY) && (hw_seq != *bufmgr_fake->last_dispatch))
385 busy_count = 0;
386 else
387 busy_count++;
388 } while (kernel_lied || ret == -EAGAIN || ret == -EINTR ||
389 (ret == -EBUSY && busy_count < 5));
390
391 if (ret != 0) {
392 drmMsg("%s:%d: Error waiting for fence: %s.\n", __FILE__, __LINE__,
393 strerror(-ret));
394 abort();
395 }
396 clear_fenced(bufmgr_fake, seq);
397 }
398
399 static int
_fence_test(drm_intel_bufmgr_fake * bufmgr_fake,unsigned fence)400 _fence_test(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence)
401 {
402 /* Slight problem with wrap-around:
403 */
404 return fence == 0 || FENCE_LTE(fence, bufmgr_fake->last_fence);
405 }
406
407 /**
408 * Allocate a memory manager block for the buffer.
409 */
410 static int
alloc_block(drm_intel_bo * bo)411 alloc_block(drm_intel_bo *bo)
412 {
413 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
414 drm_intel_bufmgr_fake *bufmgr_fake= (drm_intel_bufmgr_fake *)bo->bufmgr;
415 struct block *block = (struct block *)calloc(sizeof *block, 1);
416 unsigned int align_log2 = ffs(bo_fake->alignment) - 1;
417 unsigned int sz;
418
419 if (!block)
420 return 1;
421
422 sz = (bo->size + bo_fake->alignment - 1) & ~(bo_fake->alignment - 1);
423
424 block->mem = mmAllocMem(bufmgr_fake->heap, sz, align_log2, 0);
425 if (!block->mem) {
426 free(block);
427 return 0;
428 }
429
430 DRMINITLISTHEAD(block);
431
432 /* Insert at head or at tail???
433 */
434 DRMLISTADDTAIL(block, &bufmgr_fake->lru);
435
436 block->virtual = (uint8_t *)bufmgr_fake->virtual +
437 block->mem->ofs - bufmgr_fake->low_offset;
438 block->bo = bo;
439
440 bo_fake->block = block;
441
442 return 1;
443 }
444
445 /* Release the card storage associated with buf:
446 */
free_block(drm_intel_bufmgr_fake * bufmgr_fake,struct block * block,int skip_dirty_copy)447 static void free_block(drm_intel_bufmgr_fake *bufmgr_fake, struct block *block,
448 int skip_dirty_copy)
449 {
450 drm_intel_bo_fake *bo_fake;
451 DBG("free block %p %08x %d %d\n", block, block->mem->ofs, block->on_hardware, block->fenced);
452
453 if (!block)
454 return;
455
456 bo_fake = (drm_intel_bo_fake *)block->bo;
457
458 if (bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE))
459 skip_dirty_copy = 1;
460
461 if (!skip_dirty_copy && (bo_fake->card_dirty == 1)) {
462 memcpy(bo_fake->backing_store, block->virtual, block->bo->size);
463 bo_fake->card_dirty = 0;
464 bo_fake->dirty = 1;
465 }
466
467 if (block->on_hardware) {
468 block->bo = NULL;
469 }
470 else if (block->fenced) {
471 block->bo = NULL;
472 }
473 else {
474 DBG(" - free immediately\n");
475 DRMLISTDEL(block);
476
477 mmFreeMem(block->mem);
478 free(block);
479 }
480 }
481
482 static void
alloc_backing_store(drm_intel_bo * bo)483 alloc_backing_store(drm_intel_bo *bo)
484 {
485 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
486 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
487 assert(!bo_fake->backing_store);
488 assert(!(bo_fake->flags & (BM_PINNED|BM_NO_BACKING_STORE)));
489
490 bo_fake->backing_store = malloc(bo->size);
491
492 DBG("alloc_backing - buf %d %p %d\n", bo_fake->id, bo_fake->backing_store, bo->size);
493 assert(bo_fake->backing_store);
494 }
495
496 static void
free_backing_store(drm_intel_bo * bo)497 free_backing_store(drm_intel_bo *bo)
498 {
499 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
500
501 if (bo_fake->backing_store) {
502 assert(!(bo_fake->flags & (BM_PINNED|BM_NO_BACKING_STORE)));
503 free(bo_fake->backing_store);
504 bo_fake->backing_store = NULL;
505 }
506 }
507
508 static void
set_dirty(drm_intel_bo * bo)509 set_dirty(drm_intel_bo *bo)
510 {
511 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
512 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
513
514 if (bo_fake->flags & BM_NO_BACKING_STORE && bo_fake->invalidate_cb != NULL)
515 bo_fake->invalidate_cb(bo, bo_fake->invalidate_ptr);
516
517 assert(!(bo_fake->flags & BM_PINNED));
518
519 DBG("set_dirty - buf %d\n", bo_fake->id);
520 bo_fake->dirty = 1;
521 }
522
523 static int
evict_lru(drm_intel_bufmgr_fake * bufmgr_fake,unsigned int max_fence)524 evict_lru(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int max_fence)
525 {
526 struct block *block, *tmp;
527
528 DBG("%s\n", __FUNCTION__);
529
530 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {
531 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)block->bo;
532
533 if (bo_fake != NULL && (bo_fake->flags & BM_NO_FENCE_SUBDATA))
534 continue;
535
536 if (block->fence && max_fence && !FENCE_LTE(block->fence, max_fence))
537 return 0;
538
539 set_dirty(&bo_fake->bo);
540 bo_fake->block = NULL;
541
542 free_block(bufmgr_fake, block, 0);
543 return 1;
544 }
545
546 return 0;
547 }
548
549 static int
evict_mru(drm_intel_bufmgr_fake * bufmgr_fake)550 evict_mru(drm_intel_bufmgr_fake *bufmgr_fake)
551 {
552 struct block *block, *tmp;
553
554 DBG("%s\n", __FUNCTION__);
555
556 DRMLISTFOREACHSAFEREVERSE(block, tmp, &bufmgr_fake->lru) {
557 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)block->bo;
558
559 if (bo_fake && (bo_fake->flags & BM_NO_FENCE_SUBDATA))
560 continue;
561
562 set_dirty(&bo_fake->bo);
563 bo_fake->block = NULL;
564
565 free_block(bufmgr_fake, block, 0);
566 return 1;
567 }
568
569 return 0;
570 }
571
572 /**
573 * Removes all objects from the fenced list older than the given fence.
574 */
clear_fenced(drm_intel_bufmgr_fake * bufmgr_fake,unsigned int fence_cookie)575 static int clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake,
576 unsigned int fence_cookie)
577 {
578 struct block *block, *tmp;
579 int ret = 0;
580
581 bufmgr_fake->last_fence = fence_cookie;
582 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->fenced) {
583 assert(block->fenced);
584
585 if (_fence_test(bufmgr_fake, block->fence)) {
586
587 block->fenced = 0;
588
589 if (!block->bo) {
590 DBG("delayed free: offset %x sz %x\n",
591 block->mem->ofs, block->mem->size);
592 DRMLISTDEL(block);
593 mmFreeMem(block->mem);
594 free(block);
595 }
596 else {
597 DBG("return to lru: offset %x sz %x\n",
598 block->mem->ofs, block->mem->size);
599 DRMLISTDEL(block);
600 DRMLISTADDTAIL(block, &bufmgr_fake->lru);
601 }
602
603 ret = 1;
604 }
605 else {
606 /* Blocks are ordered by fence, so if one fails, all from
607 * here will fail also:
608 */
609 DBG("fence not passed: offset %x sz %x %d %d \n",
610 block->mem->ofs, block->mem->size, block->fence, bufmgr_fake->last_fence);
611 break;
612 }
613 }
614
615 DBG("%s: %d\n", __FUNCTION__, ret);
616 return ret;
617 }
618
fence_blocks(drm_intel_bufmgr_fake * bufmgr_fake,unsigned fence)619 static void fence_blocks(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence)
620 {
621 struct block *block, *tmp;
622
623 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) {
624 DBG("Fence block %p (sz 0x%x ofs %x buf %p) with fence %d\n", block,
625 block->mem->size, block->mem->ofs, block->bo, fence);
626 block->fence = fence;
627
628 block->on_hardware = 0;
629 block->fenced = 1;
630
631 /* Move to tail of pending list here
632 */
633 DRMLISTDEL(block);
634 DRMLISTADDTAIL(block, &bufmgr_fake->fenced);
635 }
636
637 assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));
638 }
639
evict_and_alloc_block(drm_intel_bo * bo)640 static int evict_and_alloc_block(drm_intel_bo *bo)
641 {
642 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
643 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
644
645 assert(bo_fake->block == NULL);
646
647 /* Search for already free memory:
648 */
649 if (alloc_block(bo))
650 return 1;
651
652 /* If we're not thrashing, allow lru eviction to dig deeper into
653 * recently used textures. We'll probably be thrashing soon:
654 */
655 if (!bufmgr_fake->thrashing) {
656 while (evict_lru(bufmgr_fake, 0))
657 if (alloc_block(bo))
658 return 1;
659 }
660
661 /* Keep thrashing counter alive?
662 */
663 if (bufmgr_fake->thrashing)
664 bufmgr_fake->thrashing = 20;
665
666 /* Wait on any already pending fences - here we are waiting for any
667 * freed memory that has been submitted to hardware and fenced to
668 * become available:
669 */
670 while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) {
671 uint32_t fence = bufmgr_fake->fenced.next->fence;
672 _fence_wait_internal(bufmgr_fake, fence);
673
674 if (alloc_block(bo))
675 return 1;
676 }
677
678 if (!DRMLISTEMPTY(&bufmgr_fake->on_hardware)) {
679 while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) {
680 uint32_t fence = bufmgr_fake->fenced.next->fence;
681 _fence_wait_internal(bufmgr_fake, fence);
682 }
683
684 if (!bufmgr_fake->thrashing) {
685 DBG("thrashing\n");
686 }
687 bufmgr_fake->thrashing = 20;
688
689 if (alloc_block(bo))
690 return 1;
691 }
692
693 while (evict_mru(bufmgr_fake))
694 if (alloc_block(bo))
695 return 1;
696
697 DBG("%s 0x%x bytes failed\n", __FUNCTION__, bo->size);
698
699 return 0;
700 }
701
702 /***********************************************************************
703 * Public functions
704 */
705
706 /**
707 * Wait for hardware idle by emitting a fence and waiting for it.
708 */
709 static void
drm_intel_bufmgr_fake_wait_idle(drm_intel_bufmgr_fake * bufmgr_fake)710 drm_intel_bufmgr_fake_wait_idle(drm_intel_bufmgr_fake *bufmgr_fake)
711 {
712 unsigned int cookie;
713
714 cookie = _fence_emit_internal(bufmgr_fake);
715 _fence_wait_internal(bufmgr_fake, cookie);
716 }
717
718 /**
719 * Wait for rendering to a buffer to complete.
720 *
721 * It is assumed that the bathcbuffer which performed the rendering included
722 * the necessary flushing.
723 */
724 static void
drm_intel_fake_bo_wait_rendering_locked(drm_intel_bo * bo)725 drm_intel_fake_bo_wait_rendering_locked(drm_intel_bo *bo)
726 {
727 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
728 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
729
730 if (bo_fake->block == NULL || !bo_fake->block->fenced)
731 return;
732
733 _fence_wait_internal(bufmgr_fake, bo_fake->block->fence);
734 }
735
736 static void
drm_intel_fake_bo_wait_rendering(drm_intel_bo * bo)737 drm_intel_fake_bo_wait_rendering(drm_intel_bo *bo)
738 {
739 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
740
741 pthread_mutex_lock(&bufmgr_fake->lock);
742 drm_intel_fake_bo_wait_rendering_locked(bo);
743 pthread_mutex_unlock(&bufmgr_fake->lock);
744 }
745
746 /* Specifically ignore texture memory sharing.
747 * -- just evict everything
748 * -- and wait for idle
749 */
750 void
drm_intel_bufmgr_fake_contended_lock_take(drm_intel_bufmgr * bufmgr)751 drm_intel_bufmgr_fake_contended_lock_take(drm_intel_bufmgr *bufmgr)
752 {
753 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bufmgr;
754 struct block *block, *tmp;
755
756 pthread_mutex_lock(&bufmgr_fake->lock);
757
758 bufmgr_fake->need_fence = 1;
759 bufmgr_fake->fail = 0;
760
761 /* Wait for hardware idle. We don't know where acceleration has been
762 * happening, so we'll need to wait anyway before letting anything get
763 * put on the card again.
764 */
765 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
766
767 /* Check that we hadn't released the lock without having fenced the last
768 * set of buffers.
769 */
770 assert(DRMLISTEMPTY(&bufmgr_fake->fenced));
771 assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));
772
773 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {
774 assert(_fence_test(bufmgr_fake, block->fence));
775 set_dirty(block->bo);
776 }
777
778 pthread_mutex_unlock(&bufmgr_fake->lock);
779 }
780
781 static drm_intel_bo *
drm_intel_fake_bo_alloc(drm_intel_bufmgr * bufmgr,const char * name,unsigned long size,unsigned int alignment)782 drm_intel_fake_bo_alloc(drm_intel_bufmgr *bufmgr, const char *name,
783 unsigned long size, unsigned int alignment)
784 {
785 drm_intel_bufmgr_fake *bufmgr_fake;
786 drm_intel_bo_fake *bo_fake;
787
788 bufmgr_fake = (drm_intel_bufmgr_fake *)bufmgr;
789
790 assert(size != 0);
791
792 bo_fake = calloc(1, sizeof(*bo_fake));
793 if (!bo_fake)
794 return NULL;
795
796 bo_fake->bo.size = size;
797 bo_fake->bo.offset = -1;
798 bo_fake->bo.virtual = NULL;
799 bo_fake->bo.bufmgr = bufmgr;
800 bo_fake->refcount = 1;
801
802 /* Alignment must be a power of two */
803 assert((alignment & (alignment - 1)) == 0);
804 if (alignment == 0)
805 alignment = 1;
806 bo_fake->alignment = alignment;
807 bo_fake->id = ++bufmgr_fake->buf_nr;
808 bo_fake->name = name;
809 bo_fake->flags = 0;
810 bo_fake->is_static = 0;
811
812 DBG("drm_bo_alloc: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name,
813 bo_fake->bo.size / 1024);
814
815 return &bo_fake->bo;
816 }
817
818 drm_intel_bo *
drm_intel_bo_fake_alloc_static(drm_intel_bufmgr * bufmgr,const char * name,unsigned long offset,unsigned long size,void * virtual)819 drm_intel_bo_fake_alloc_static(drm_intel_bufmgr *bufmgr, const char *name,
820 unsigned long offset, unsigned long size,
821 void *virtual)
822 {
823 drm_intel_bufmgr_fake *bufmgr_fake;
824 drm_intel_bo_fake *bo_fake;
825
826 bufmgr_fake = (drm_intel_bufmgr_fake *)bufmgr;
827
828 assert(size != 0);
829
830 bo_fake = calloc(1, sizeof(*bo_fake));
831 if (!bo_fake)
832 return NULL;
833
834 bo_fake->bo.size = size;
835 bo_fake->bo.offset = offset;
836 bo_fake->bo.virtual = virtual;
837 bo_fake->bo.bufmgr = bufmgr;
838 bo_fake->refcount = 1;
839 bo_fake->id = ++bufmgr_fake->buf_nr;
840 bo_fake->name = name;
841 bo_fake->flags = BM_PINNED;
842 bo_fake->is_static = 1;
843
844 DBG("drm_bo_alloc_static: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name,
845 bo_fake->bo.size / 1024);
846
847 return &bo_fake->bo;
848 }
849
850 static void
drm_intel_fake_bo_reference(drm_intel_bo * bo)851 drm_intel_fake_bo_reference(drm_intel_bo *bo)
852 {
853 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
854 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
855
856 pthread_mutex_lock(&bufmgr_fake->lock);
857 bo_fake->refcount++;
858 pthread_mutex_unlock(&bufmgr_fake->lock);
859 }
860
861 static void
drm_intel_fake_bo_reference_locked(drm_intel_bo * bo)862 drm_intel_fake_bo_reference_locked(drm_intel_bo *bo)
863 {
864 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
865
866 bo_fake->refcount++;
867 }
868
869 static void
drm_intel_fake_bo_unreference_locked(drm_intel_bo * bo)870 drm_intel_fake_bo_unreference_locked(drm_intel_bo *bo)
871 {
872 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
873 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
874 int i;
875
876 if (--bo_fake->refcount == 0) {
877 assert(bo_fake->map_count == 0);
878 /* No remaining references, so free it */
879 if (bo_fake->block)
880 free_block(bufmgr_fake, bo_fake->block, 1);
881 free_backing_store(bo);
882
883 for (i = 0; i < bo_fake->nr_relocs; i++)
884 drm_intel_fake_bo_unreference_locked(bo_fake->relocs[i].target_buf);
885
886 DBG("drm_bo_unreference: free buf %d %s\n", bo_fake->id, bo_fake->name);
887
888 free(bo_fake->relocs);
889 free(bo);
890 }
891 }
892
893 static void
drm_intel_fake_bo_unreference(drm_intel_bo * bo)894 drm_intel_fake_bo_unreference(drm_intel_bo *bo)
895 {
896 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
897
898 pthread_mutex_lock(&bufmgr_fake->lock);
899 drm_intel_fake_bo_unreference_locked(bo);
900 pthread_mutex_unlock(&bufmgr_fake->lock);
901 }
902
903 /**
904 * Set the buffer as not requiring backing store, and instead get the callback
905 * invoked whenever it would be set dirty.
906 */
drm_intel_bo_fake_disable_backing_store(drm_intel_bo * bo,void (* invalidate_cb)(drm_intel_bo * bo,void * ptr),void * ptr)907 void drm_intel_bo_fake_disable_backing_store(drm_intel_bo *bo,
908 void (*invalidate_cb)(drm_intel_bo *bo,
909 void *ptr),
910 void *ptr)
911 {
912 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
913 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
914
915 pthread_mutex_lock(&bufmgr_fake->lock);
916
917 if (bo_fake->backing_store)
918 free_backing_store(bo);
919
920 bo_fake->flags |= BM_NO_BACKING_STORE;
921
922 DBG("disable_backing_store set buf %d dirty\n", bo_fake->id);
923 bo_fake->dirty = 1;
924 bo_fake->invalidate_cb = invalidate_cb;
925 bo_fake->invalidate_ptr = ptr;
926
927 /* Note that it is invalid right from the start. Also note
928 * invalidate_cb is called with the bufmgr locked, so cannot
929 * itself make bufmgr calls.
930 */
931 if (invalidate_cb != NULL)
932 invalidate_cb(bo, ptr);
933
934 pthread_mutex_unlock(&bufmgr_fake->lock);
935 }
936
937 /**
938 * Map a buffer into bo->virtual, allocating either card memory space (If
939 * BM_NO_BACKING_STORE or BM_PINNED) or backing store, as necessary.
940 */
941 static int
drm_intel_fake_bo_map_locked(drm_intel_bo * bo,int write_enable)942 drm_intel_fake_bo_map_locked(drm_intel_bo *bo, int write_enable)
943 {
944 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
945 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
946
947 /* Static buffers are always mapped. */
948 if (bo_fake->is_static) {
949 if (bo_fake->card_dirty) {
950 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
951 bo_fake->card_dirty = 0;
952 }
953 return 0;
954 }
955
956 /* Allow recursive mapping. Mesa may recursively map buffers with
957 * nested display loops, and it is used internally in bufmgr_fake
958 * for relocation.
959 */
960 if (bo_fake->map_count++ != 0)
961 return 0;
962
963 {
964 DBG("drm_bo_map: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name,
965 bo_fake->bo.size / 1024);
966
967 if (bo->virtual != NULL) {
968 drmMsg("%s: already mapped\n", __FUNCTION__);
969 abort();
970 }
971 else if (bo_fake->flags & (BM_NO_BACKING_STORE|BM_PINNED)) {
972
973 if (!bo_fake->block && !evict_and_alloc_block(bo)) {
974 DBG("%s: alloc failed\n", __FUNCTION__);
975 bufmgr_fake->fail = 1;
976 return 1;
977 }
978 else {
979 assert(bo_fake->block);
980 bo_fake->dirty = 0;
981
982 if (!(bo_fake->flags & BM_NO_FENCE_SUBDATA) &&
983 bo_fake->block->fenced) {
984 drm_intel_fake_bo_wait_rendering_locked(bo);
985 }
986
987 bo->virtual = bo_fake->block->virtual;
988 }
989 }
990 else {
991 if (write_enable)
992 set_dirty(bo);
993
994 if (bo_fake->backing_store == 0)
995 alloc_backing_store(bo);
996
997 if ((bo_fake->card_dirty == 1) && bo_fake->block) {
998 if (bo_fake->block->fenced)
999 drm_intel_fake_bo_wait_rendering_locked(bo);
1000
1001 memcpy(bo_fake->backing_store, bo_fake->block->virtual, bo_fake->block->bo->size);
1002 bo_fake->card_dirty = 0;
1003 }
1004
1005 bo->virtual = bo_fake->backing_store;
1006 }
1007 }
1008
1009 return 0;
1010 }
1011
1012 static int
drm_intel_fake_bo_map(drm_intel_bo * bo,int write_enable)1013 drm_intel_fake_bo_map(drm_intel_bo *bo, int write_enable)
1014 {
1015 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
1016 int ret;
1017
1018 pthread_mutex_lock(&bufmgr_fake->lock);
1019 ret = drm_intel_fake_bo_map_locked(bo, write_enable);
1020 pthread_mutex_unlock(&bufmgr_fake->lock);
1021
1022 return ret;
1023 }
1024
1025 static int
drm_intel_fake_bo_unmap_locked(drm_intel_bo * bo)1026 drm_intel_fake_bo_unmap_locked(drm_intel_bo *bo)
1027 {
1028 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
1029 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
1030
1031 /* Static buffers are always mapped. */
1032 if (bo_fake->is_static)
1033 return 0;
1034
1035 assert(bo_fake->map_count != 0);
1036 if (--bo_fake->map_count != 0)
1037 return 0;
1038
1039 DBG("drm_bo_unmap: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name,
1040 bo_fake->bo.size / 1024);
1041
1042 bo->virtual = NULL;
1043
1044 return 0;
1045 }
1046
1047 static int
drm_intel_fake_bo_unmap(drm_intel_bo * bo)1048 drm_intel_fake_bo_unmap(drm_intel_bo *bo)
1049 {
1050 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
1051 int ret;
1052
1053 pthread_mutex_lock(&bufmgr_fake->lock);
1054 ret = drm_intel_fake_bo_unmap_locked(bo);
1055 pthread_mutex_unlock(&bufmgr_fake->lock);
1056
1057 return ret;
1058 }
1059
1060 static void
drm_intel_fake_kick_all_locked(drm_intel_bufmgr_fake * bufmgr_fake)1061 drm_intel_fake_kick_all_locked(drm_intel_bufmgr_fake *bufmgr_fake)
1062 {
1063 struct block *block, *tmp;
1064
1065 bufmgr_fake->performed_rendering = 0;
1066 /* okay for ever BO that is on the HW kick it off.
1067 seriously not afraid of the POLICE right now */
1068 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) {
1069 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)block->bo;
1070
1071 block->on_hardware = 0;
1072 free_block(bufmgr_fake, block, 0);
1073 bo_fake->block = NULL;
1074 bo_fake->validated = 0;
1075 if (!(bo_fake->flags & BM_NO_BACKING_STORE))
1076 bo_fake->dirty = 1;
1077 }
1078
1079 }
1080
1081 static int
drm_intel_fake_bo_validate(drm_intel_bo * bo)1082 drm_intel_fake_bo_validate(drm_intel_bo *bo)
1083 {
1084 drm_intel_bufmgr_fake *bufmgr_fake;
1085 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
1086
1087 bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
1088
1089 DBG("drm_bo_validate: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name,
1090 bo_fake->bo.size / 1024);
1091
1092 /* Sanity check: Buffers should be unmapped before being validated.
1093 * This is not so much of a problem for bufmgr_fake, but TTM refuses,
1094 * and the problem is harder to debug there.
1095 */
1096 assert(bo_fake->map_count == 0);
1097
1098 if (bo_fake->is_static) {
1099 /* Add it to the needs-fence list */
1100 bufmgr_fake->need_fence = 1;
1101 return 0;
1102 }
1103
1104 /* Allocate the card memory */
1105 if (!bo_fake->block && !evict_and_alloc_block(bo)) {
1106 bufmgr_fake->fail = 1;
1107 DBG("Failed to validate buf %d:%s\n", bo_fake->id, bo_fake->name);
1108 return -1;
1109 }
1110
1111 assert(bo_fake->block);
1112 assert(bo_fake->block->bo == &bo_fake->bo);
1113
1114 bo->offset = bo_fake->block->mem->ofs;
1115
1116 /* Upload the buffer contents if necessary */
1117 if (bo_fake->dirty) {
1118 DBG("Upload dirty buf %d:%s, sz %d offset 0x%x\n", bo_fake->id,
1119 bo_fake->name, bo->size, bo_fake->block->mem->ofs);
1120
1121 assert(!(bo_fake->flags &
1122 (BM_NO_BACKING_STORE|BM_PINNED)));
1123
1124 /* Actually, should be able to just wait for a fence on the memory,
1125 * which we would be tracking when we free it. Waiting for idle is
1126 * a sufficiently large hammer for now.
1127 */
1128 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
1129
1130 /* we may never have mapped this BO so it might not have any backing
1131 * store if this happens it should be rare, but 0 the card memory
1132 * in any case */
1133 if (bo_fake->backing_store)
1134 memcpy(bo_fake->block->virtual, bo_fake->backing_store, bo->size);
1135 else
1136 memset(bo_fake->block->virtual, 0, bo->size);
1137
1138 bo_fake->dirty = 0;
1139 }
1140
1141 bo_fake->block->fenced = 0;
1142 bo_fake->block->on_hardware = 1;
1143 DRMLISTDEL(bo_fake->block);
1144 DRMLISTADDTAIL(bo_fake->block, &bufmgr_fake->on_hardware);
1145
1146 bo_fake->validated = 1;
1147 bufmgr_fake->need_fence = 1;
1148
1149 return 0;
1150 }
1151
1152 static void
drm_intel_fake_fence_validated(drm_intel_bufmgr * bufmgr)1153 drm_intel_fake_fence_validated(drm_intel_bufmgr *bufmgr)
1154 {
1155 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bufmgr;
1156 unsigned int cookie;
1157
1158 cookie = _fence_emit_internal(bufmgr_fake);
1159 fence_blocks(bufmgr_fake, cookie);
1160
1161 DBG("drm_fence_validated: 0x%08x cookie\n", cookie);
1162 }
1163
1164 static void
drm_intel_fake_destroy(drm_intel_bufmgr * bufmgr)1165 drm_intel_fake_destroy(drm_intel_bufmgr *bufmgr)
1166 {
1167 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bufmgr;
1168
1169 pthread_mutex_destroy(&bufmgr_fake->lock);
1170 mmDestroy(bufmgr_fake->heap);
1171 free(bufmgr);
1172 }
1173
1174 static int
drm_intel_fake_emit_reloc(drm_intel_bo * bo,uint32_t offset,drm_intel_bo * target_bo,uint32_t target_offset,uint32_t read_domains,uint32_t write_domain)1175 drm_intel_fake_emit_reloc(drm_intel_bo *bo, uint32_t offset,
1176 drm_intel_bo *target_bo, uint32_t target_offset,
1177 uint32_t read_domains, uint32_t write_domain)
1178 {
1179 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
1180 struct fake_buffer_reloc *r;
1181 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
1182 drm_intel_bo_fake *target_fake = (drm_intel_bo_fake *)target_bo;
1183 int i;
1184
1185 pthread_mutex_lock(&bufmgr_fake->lock);
1186
1187 assert(bo);
1188 assert(target_bo);
1189
1190 if (bo_fake->relocs == NULL) {
1191 bo_fake->relocs = malloc(sizeof(struct fake_buffer_reloc) * MAX_RELOCS);
1192 }
1193
1194 r = &bo_fake->relocs[bo_fake->nr_relocs++];
1195
1196 assert(bo_fake->nr_relocs <= MAX_RELOCS);
1197
1198 drm_intel_fake_bo_reference_locked(target_bo);
1199
1200 if (!target_fake->is_static) {
1201 bo_fake->child_size += ALIGN(target_bo->size, target_fake->alignment);
1202 bo_fake->child_size += target_fake->child_size;
1203 }
1204 r->target_buf = target_bo;
1205 r->offset = offset;
1206 r->last_target_offset = target_bo->offset;
1207 r->delta = target_offset;
1208 r->read_domains = read_domains;
1209 r->write_domain = write_domain;
1210
1211 if (bufmgr_fake->debug) {
1212 /* Check that a conflicting relocation hasn't already been emitted. */
1213 for (i = 0; i < bo_fake->nr_relocs - 1; i++) {
1214 struct fake_buffer_reloc *r2 = &bo_fake->relocs[i];
1215
1216 assert(r->offset != r2->offset);
1217 }
1218 }
1219
1220 pthread_mutex_unlock(&bufmgr_fake->lock);
1221
1222 return 0;
1223 }
1224
1225 /**
1226 * Incorporates the validation flags associated with each relocation into
1227 * the combined validation flags for the buffer on this batchbuffer submission.
1228 */
1229 static void
drm_intel_fake_calculate_domains(drm_intel_bo * bo)1230 drm_intel_fake_calculate_domains(drm_intel_bo *bo)
1231 {
1232 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
1233 int i;
1234
1235 for (i = 0; i < bo_fake->nr_relocs; i++) {
1236 struct fake_buffer_reloc *r = &bo_fake->relocs[i];
1237 drm_intel_bo_fake *target_fake = (drm_intel_bo_fake *)r->target_buf;
1238
1239 /* Do the same for the tree of buffers we depend on */
1240 drm_intel_fake_calculate_domains(r->target_buf);
1241
1242 target_fake->read_domains |= r->read_domains;
1243 target_fake->write_domain |= r->write_domain;
1244 }
1245 }
1246
1247
1248 static int
drm_intel_fake_reloc_and_validate_buffer(drm_intel_bo * bo)1249 drm_intel_fake_reloc_and_validate_buffer(drm_intel_bo *bo)
1250 {
1251 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
1252 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
1253 int i, ret;
1254
1255 assert(bo_fake->map_count == 0);
1256
1257 for (i = 0; i < bo_fake->nr_relocs; i++) {
1258 struct fake_buffer_reloc *r = &bo_fake->relocs[i];
1259 drm_intel_bo_fake *target_fake = (drm_intel_bo_fake *)r->target_buf;
1260 uint32_t reloc_data;
1261
1262 /* Validate the target buffer if that hasn't been done. */
1263 if (!target_fake->validated) {
1264 ret = drm_intel_fake_reloc_and_validate_buffer(r->target_buf);
1265 if (ret != 0) {
1266 if (bo->virtual != NULL)
1267 drm_intel_fake_bo_unmap_locked(bo);
1268 return ret;
1269 }
1270 }
1271
1272 /* Calculate the value of the relocation entry. */
1273 if (r->target_buf->offset != r->last_target_offset) {
1274 reloc_data = r->target_buf->offset + r->delta;
1275
1276 if (bo->virtual == NULL)
1277 drm_intel_fake_bo_map_locked(bo, 1);
1278
1279 *(uint32_t *)((uint8_t *)bo->virtual + r->offset) = reloc_data;
1280
1281 r->last_target_offset = r->target_buf->offset;
1282 }
1283 }
1284
1285 if (bo->virtual != NULL)
1286 drm_intel_fake_bo_unmap_locked(bo);
1287
1288 if (bo_fake->write_domain != 0) {
1289 if (!(bo_fake->flags & (BM_NO_BACKING_STORE|BM_PINNED))) {
1290 if (bo_fake->backing_store == 0)
1291 alloc_backing_store(bo);
1292 }
1293 bo_fake->card_dirty = 1;
1294 bufmgr_fake->performed_rendering = 1;
1295 }
1296
1297 return drm_intel_fake_bo_validate(bo);
1298 }
1299
1300 static void
drm_intel_bo_fake_post_submit(drm_intel_bo * bo)1301 drm_intel_bo_fake_post_submit(drm_intel_bo *bo)
1302 {
1303 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
1304 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo;
1305 int i;
1306
1307 for (i = 0; i < bo_fake->nr_relocs; i++) {
1308 struct fake_buffer_reloc *r = &bo_fake->relocs[i];
1309 drm_intel_bo_fake *target_fake = (drm_intel_bo_fake *)r->target_buf;
1310
1311 if (target_fake->validated)
1312 drm_intel_bo_fake_post_submit(r->target_buf);
1313
1314 DBG("%s@0x%08x + 0x%08x -> %s@0x%08x + 0x%08x\n",
1315 bo_fake->name, (uint32_t)bo->offset, r->offset,
1316 target_fake->name, (uint32_t)r->target_buf->offset, r->delta);
1317 }
1318
1319 assert(bo_fake->map_count == 0);
1320 bo_fake->validated = 0;
1321 bo_fake->read_domains = 0;
1322 bo_fake->write_domain = 0;
1323 }
1324
1325
drm_intel_bufmgr_fake_set_exec_callback(drm_intel_bufmgr * bufmgr,int (* exec)(drm_intel_bo * bo,unsigned int used,void * priv),void * priv)1326 void drm_intel_bufmgr_fake_set_exec_callback(drm_intel_bufmgr *bufmgr,
1327 int (*exec)(drm_intel_bo *bo,
1328 unsigned int used,
1329 void *priv),
1330 void *priv)
1331 {
1332 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bufmgr;
1333
1334 bufmgr_fake->exec = exec;
1335 bufmgr_fake->exec_priv = priv;
1336 }
1337
1338 static int
drm_intel_fake_bo_exec(drm_intel_bo * bo,int used,drm_clip_rect_t * cliprects,int num_cliprects,int DR4)1339 drm_intel_fake_bo_exec(drm_intel_bo *bo, int used,
1340 drm_clip_rect_t *cliprects, int num_cliprects,
1341 int DR4)
1342 {
1343 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo->bufmgr;
1344 drm_intel_bo_fake *batch_fake = (drm_intel_bo_fake *)bo;
1345 struct drm_i915_batchbuffer batch;
1346 int ret;
1347 int retry_count = 0;
1348
1349 pthread_mutex_lock(&bufmgr_fake->lock);
1350
1351 bufmgr_fake->performed_rendering = 0;
1352
1353 drm_intel_fake_calculate_domains(bo);
1354
1355 batch_fake->read_domains = I915_GEM_DOMAIN_COMMAND;
1356
1357 /* we've ran out of RAM so blow the whole lot away and retry */
1358 restart:
1359 ret = drm_intel_fake_reloc_and_validate_buffer(bo);
1360 if (bufmgr_fake->fail == 1) {
1361 if (retry_count == 0) {
1362 retry_count++;
1363 drm_intel_fake_kick_all_locked(bufmgr_fake);
1364 bufmgr_fake->fail = 0;
1365 goto restart;
1366 } else /* dump out the memory here */
1367 mmDumpMemInfo(bufmgr_fake->heap);
1368 }
1369
1370 assert(ret == 0);
1371
1372 if (bufmgr_fake->exec != NULL) {
1373 int ret = bufmgr_fake->exec(bo, used, bufmgr_fake->exec_priv);
1374 if (ret != 0) {
1375 pthread_mutex_unlock(&bufmgr_fake->lock);
1376 return ret;
1377 }
1378 } else {
1379 batch.start = bo->offset;
1380 batch.used = used;
1381 batch.cliprects = cliprects;
1382 batch.num_cliprects = num_cliprects;
1383 batch.DR1 = 0;
1384 batch.DR4 = DR4;
1385
1386 if (drmCommandWrite(bufmgr_fake->fd, DRM_I915_BATCHBUFFER, &batch,
1387 sizeof(batch))) {
1388 drmMsg("DRM_I915_BATCHBUFFER: %d\n", -errno);
1389 pthread_mutex_unlock(&bufmgr_fake->lock);
1390 return -errno;
1391 }
1392 }
1393
1394 drm_intel_fake_fence_validated(bo->bufmgr);
1395
1396 drm_intel_bo_fake_post_submit(bo);
1397
1398 pthread_mutex_unlock(&bufmgr_fake->lock);
1399
1400 return 0;
1401 }
1402
1403 /**
1404 * Return an error if the list of BOs will exceed the aperture size.
1405 *
1406 * This is a rough guess and likely to fail, as during the validate sequence we
1407 * may place a buffer in an inopportune spot early on and then fail to fit
1408 * a set smaller than the aperture.
1409 */
1410 static int
drm_intel_fake_check_aperture_space(drm_intel_bo ** bo_array,int count)1411 drm_intel_fake_check_aperture_space(drm_intel_bo **bo_array, int count)
1412 {
1413 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bo_array[0]->bufmgr;
1414 unsigned int sz = 0;
1415 int i;
1416
1417 for (i = 0; i < count; i++) {
1418 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)bo_array[i];
1419
1420 if (bo_fake == NULL)
1421 continue;
1422
1423 if (!bo_fake->is_static)
1424 sz += ALIGN(bo_array[i]->size, bo_fake->alignment);
1425 sz += bo_fake->child_size;
1426 }
1427
1428 if (sz > bufmgr_fake->size) {
1429 DBG("check_space: overflowed bufmgr size, %dkb vs %dkb\n",
1430 sz / 1024, bufmgr_fake->size / 1024);
1431 return -1;
1432 }
1433
1434 DBG("drm_check_space: sz %dkb vs bufgr %dkb\n", sz / 1024 ,
1435 bufmgr_fake->size / 1024);
1436 return 0;
1437 }
1438
1439 /**
1440 * Evicts all buffers, waiting for fences to pass and copying contents out
1441 * as necessary.
1442 *
1443 * Used by the X Server on LeaveVT, when the card memory is no longer our
1444 * own.
1445 */
1446 void
drm_intel_bufmgr_fake_evict_all(drm_intel_bufmgr * bufmgr)1447 drm_intel_bufmgr_fake_evict_all(drm_intel_bufmgr *bufmgr)
1448 {
1449 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bufmgr;
1450 struct block *block, *tmp;
1451
1452 pthread_mutex_lock(&bufmgr_fake->lock);
1453
1454 bufmgr_fake->need_fence = 1;
1455 bufmgr_fake->fail = 0;
1456
1457 /* Wait for hardware idle. We don't know where acceleration has been
1458 * happening, so we'll need to wait anyway before letting anything get
1459 * put on the card again.
1460 */
1461 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
1462
1463 /* Check that we hadn't released the lock without having fenced the last
1464 * set of buffers.
1465 */
1466 assert(DRMLISTEMPTY(&bufmgr_fake->fenced));
1467 assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));
1468
1469 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {
1470 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *)block->bo;
1471 /* Releases the memory, and memcpys dirty contents out if necessary. */
1472 free_block(bufmgr_fake, block, 0);
1473 bo_fake->block = NULL;
1474 }
1475
1476 pthread_mutex_unlock(&bufmgr_fake->lock);
1477 }
drm_intel_bufmgr_fake_set_last_dispatch(drm_intel_bufmgr * bufmgr,volatile unsigned int * last_dispatch)1478 void drm_intel_bufmgr_fake_set_last_dispatch(drm_intel_bufmgr *bufmgr,
1479 volatile unsigned int *last_dispatch)
1480 {
1481 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *)bufmgr;
1482
1483 bufmgr_fake->last_dispatch = (volatile int *)last_dispatch;
1484 }
1485
1486 drm_intel_bufmgr *
drm_intel_bufmgr_fake_init(int fd,unsigned long low_offset,void * low_virtual,unsigned long size,volatile unsigned int * last_dispatch)1487 drm_intel_bufmgr_fake_init(int fd,
1488 unsigned long low_offset, void *low_virtual,
1489 unsigned long size,
1490 volatile unsigned int *last_dispatch)
1491 {
1492 drm_intel_bufmgr_fake *bufmgr_fake;
1493
1494 bufmgr_fake = calloc(1, sizeof(*bufmgr_fake));
1495
1496 if (pthread_mutex_init(&bufmgr_fake->lock, NULL) != 0) {
1497 free(bufmgr_fake);
1498 return NULL;
1499 }
1500
1501 /* Initialize allocator */
1502 DRMINITLISTHEAD(&bufmgr_fake->fenced);
1503 DRMINITLISTHEAD(&bufmgr_fake->on_hardware);
1504 DRMINITLISTHEAD(&bufmgr_fake->lru);
1505
1506 bufmgr_fake->low_offset = low_offset;
1507 bufmgr_fake->virtual = low_virtual;
1508 bufmgr_fake->size = size;
1509 bufmgr_fake->heap = mmInit(low_offset, size);
1510
1511 /* Hook in methods */
1512 bufmgr_fake->bufmgr.bo_alloc = drm_intel_fake_bo_alloc;
1513 bufmgr_fake->bufmgr.bo_alloc_for_render = drm_intel_fake_bo_alloc;
1514 bufmgr_fake->bufmgr.bo_reference = drm_intel_fake_bo_reference;
1515 bufmgr_fake->bufmgr.bo_unreference = drm_intel_fake_bo_unreference;
1516 bufmgr_fake->bufmgr.bo_map = drm_intel_fake_bo_map;
1517 bufmgr_fake->bufmgr.bo_unmap = drm_intel_fake_bo_unmap;
1518 bufmgr_fake->bufmgr.bo_wait_rendering = drm_intel_fake_bo_wait_rendering;
1519 bufmgr_fake->bufmgr.bo_emit_reloc = drm_intel_fake_emit_reloc;
1520 bufmgr_fake->bufmgr.destroy = drm_intel_fake_destroy;
1521 bufmgr_fake->bufmgr.bo_exec = drm_intel_fake_bo_exec;
1522 bufmgr_fake->bufmgr.check_aperture_space = drm_intel_fake_check_aperture_space;
1523 bufmgr_fake->bufmgr.debug = 0;
1524
1525 bufmgr_fake->fd = fd;
1526 bufmgr_fake->last_dispatch = (volatile int *)last_dispatch;
1527
1528 return &bufmgr_fake->bufmgr;
1529 }
1530
1531