1 /**************************************************************************
2 *
3 * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA
4 * All Rights Reserved.
5 * Copyright 2009 VMware, Inc., Palo Alto, CA., USA
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
23 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
24 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
25 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
26 * USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
28 **************************************************************************/
29 /*
30 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
31 */
32
33 #ifdef HAVE_CONFIG_H
34 #include "config.h"
35 #endif
36
37 #include <drm/psb_ttm_placement_user.h>
38 #include <stdint.h>
39 #include <sys/time.h>
40 #include <errno.h>
41 #include <unistd.h>
42 #include <assert.h>
43 #include <string.h>
44 #include <sys/mman.h>
45 #include <xf86drm.h>
46 #include "wsbm_pool.h"
47 #include "wsbm_fencemgr.h"
48 #include "wsbm_priv.h"
49 #include "wsbm_manager.h"
50
51 #define WSBM_SLABPOOL_ALLOC_RETRIES 100
52 #define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \
53 do { \
54 (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \
55 } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \
56
57 #define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \
58 do { \
59 (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \
60 } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \
61
62
63 #ifdef DEBUG_FENCESIGNALED
64 static int createbuffer = 0;
65 static int fencesignaled = 0;
66 #endif
67
68 struct _WsbmSlab;
69
70 struct _WsbmSlabBuffer
71 {
72 struct _WsbmKernelBuf kBuf;
73 struct _WsbmBufStorage storage;
74 struct _WsbmCond event;
75
76 /*
77 * Remains constant after creation.
78 */
79
80 int isSlabBuffer;
81 struct _WsbmSlab *parent;
82 uint32_t start;
83 void *virtual;
84 unsigned long requestedSize;
85 uint64_t mapHandle;
86
87 /*
88 * Protected by struct _WsbmSlabSizeHeader::mutex
89 */
90
91 struct _WsbmListHead head;
92
93 /*
94 * Protected by this::mutex
95 */
96
97 struct _WsbmFenceObject *fence;
98 uint32_t fenceType;
99 struct _WsbmAtomic writers; /* (Only upping) */
100 int unFenced;
101 };
102
103 struct _WsbmSlabPool;
104 struct _WsbmSlabKernelBO
105 {
106
107 /*
108 * Constant at creation
109 */
110
111 struct _WsbmKernelBuf kBuf;
112 uint32_t pageAlignment;
113 void *virtual;
114 unsigned long actualSize;
115 uint64_t mapHandle;
116
117 /*
118 * Protected by struct _WsbmSlabCache::mutex
119 */
120
121 struct _WsbmSlabPool *slabPool;
122 uint32_t proposedPlacement;
123 struct _WsbmListHead timeoutHead;
124 struct _WsbmListHead head;
125 struct timeval timeFreed;
126 };
127
128 struct _WsbmSlab
129 {
130 struct _WsbmListHead head;
131 struct _WsbmListHead freeBuffers;
132 uint32_t numBuffers;
133 uint32_t numFree;
134 struct _WsbmSlabBuffer *buffers;
135 struct _WsbmSlabSizeHeader *header;
136 struct _WsbmSlabKernelBO *kbo;
137 };
138
139 struct _WsbmSlabSizeHeader
140 {
141 /*
142 * Constant at creation.
143 */
144 struct _WsbmSlabPool *slabPool;
145 uint32_t bufSize;
146
147 /*
148 * Protected by this::mutex
149 */
150
151 struct _WsbmListHead slabs;
152 struct _WsbmListHead freeSlabs;
153 struct _WsbmListHead delayedBuffers;
154 uint32_t numDelayed;
155 struct _WsbmMutex mutex;
156 };
157
158 struct _WsbmSlabCache
159 {
160 struct timeval slabTimeout;
161 struct timeval checkInterval;
162 struct timeval nextCheck;
163 struct _WsbmListHead timeoutList;
164 struct _WsbmListHead unCached;
165 struct _WsbmListHead cached;
166 struct _WsbmMutex mutex;
167 };
168
169 struct _WsbmSlabPool
170 {
171 struct _WsbmBufferPool pool;
172
173 /*
174 * The data of this structure remains constant after
175 * initialization and thus needs no mutex protection.
176 */
177
178 unsigned int devOffset;
179 struct _WsbmSlabCache *cache;
180 uint32_t proposedPlacement;
181 uint32_t validMask;
182 uint32_t *bucketSizes;
183 uint32_t numBuckets;
184 uint32_t pageSize;
185 int pageAlignment;
186 int maxSlabSize;
187 int desiredNumBuffers;
188 struct _WsbmSlabSizeHeader *headers;
189 };
190
191 static inline struct _WsbmSlabPool *
slabPoolFromPool(struct _WsbmBufferPool * pool)192 slabPoolFromPool(struct _WsbmBufferPool *pool)
193 {
194 return containerOf(pool, struct _WsbmSlabPool, pool);
195 }
196
197 static inline struct _WsbmSlabPool *
slabPoolFromBuf(struct _WsbmSlabBuffer * sBuf)198 slabPoolFromBuf(struct _WsbmSlabBuffer *sBuf)
199 {
200 return slabPoolFromPool(sBuf->storage.pool);
201 }
202
203 static inline struct _WsbmSlabBuffer *
slabBuffer(struct _WsbmBufStorage * buf)204 slabBuffer(struct _WsbmBufStorage *buf)
205 {
206 return containerOf(buf, struct _WsbmSlabBuffer, storage);
207 }
208
209 /*
210 * FIXME: Perhaps arrange timeout slabs in size buckets for fast
211 * retreival??
212 */
213
214 static inline int
wsbmTimeAfterEq(struct timeval * arg1,struct timeval * arg2)215 wsbmTimeAfterEq(struct timeval *arg1, struct timeval *arg2)
216 {
217 return ((arg1->tv_sec > arg2->tv_sec) ||
218 ((arg1->tv_sec == arg2->tv_sec) &&
219 (arg1->tv_usec > arg2->tv_usec)));
220 }
221
222 static inline void
wsbmTimeAdd(struct timeval * arg,struct timeval * add)223 wsbmTimeAdd(struct timeval *arg, struct timeval *add)
224 {
225 unsigned int sec;
226
227 arg->tv_sec += add->tv_sec;
228 arg->tv_usec += add->tv_usec;
229 sec = arg->tv_usec / 1000000;
230 arg->tv_sec += sec;
231 arg->tv_usec -= sec * 1000000;
232 }
233
234 static void
wsbmFreeKernelBO(struct _WsbmSlabKernelBO * kbo)235 wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo)
236 {
237 struct ttm_pl_reference_req arg;
238 struct _WsbmSlabPool *slabPool;
239
240 if (!kbo)
241 return;
242
243 slabPool = kbo->slabPool;
244 arg.handle = kbo->kBuf.handle;
245 (void)munmap(kbo->virtual, kbo->actualSize);
246 (void)drmCommandWrite(slabPool->pool.fd,
247 slabPool->devOffset + TTM_PL_UNREF, &arg,
248 sizeof(arg));
249 free(kbo);
250 }
251
252 static void
wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache * cache,struct timeval * time)253 wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache *cache, struct timeval *time)
254 {
255 struct _WsbmListHead *list, *next;
256 struct _WsbmSlabKernelBO *kbo;
257
258 if (!wsbmTimeAfterEq(time, &cache->nextCheck))
259 return;
260
261 WSBMLISTFOREACHSAFE(list, next, &cache->timeoutList) {
262 kbo = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, timeoutHead);
263
264 if (!wsbmTimeAfterEq(time, &kbo->timeFreed))
265 break;
266
267 WSBMLISTDELINIT(&kbo->timeoutHead);
268 WSBMLISTDELINIT(&kbo->head);
269 wsbmFreeKernelBO(kbo);
270 }
271
272 cache->nextCheck = *time;
273 wsbmTimeAdd(&cache->nextCheck, &cache->checkInterval);
274 }
275
276 /*
277 * Add a _SlabKernelBO to the free slab manager.
278 * This means that it is available for reuse, but if it's not
279 * reused in a while, it will be freed.
280 */
281
282 static void
wsbmSetKernelBOFree(struct _WsbmSlabCache * cache,struct _WsbmSlabKernelBO * kbo)283 wsbmSetKernelBOFree(struct _WsbmSlabCache *cache,
284 struct _WsbmSlabKernelBO *kbo)
285 {
286 struct timeval time;
287 struct timeval timeFreed;
288
289 gettimeofday(&time, NULL);
290 timeFreed = time;
291 WSBM_MUTEX_LOCK(&cache->mutex);
292 wsbmTimeAdd(&timeFreed, &cache->slabTimeout);
293 kbo->timeFreed = timeFreed;
294
295 if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED)
296 WSBMLISTADD(&kbo->head, &cache->cached);
297 else
298 WSBMLISTADD(&kbo->head, &cache->unCached);
299
300 WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList);
301 wsbmFreeTimeoutKBOsLocked(cache, &time);
302
303 WSBM_MUTEX_UNLOCK(&cache->mutex);
304 }
305
306 /*
307 * Get a _SlabKernelBO for us to use as storage for a slab.
308 */
309
310 static struct _WsbmSlabKernelBO *
wsbmAllocKernelBO(struct _WsbmSlabSizeHeader * header)311 wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header)
312 {
313 struct _WsbmSlabPool *slabPool = header->slabPool;
314 struct _WsbmSlabCache *cache = slabPool->cache;
315 struct _WsbmListHead *list, *head;
316 uint32_t size = header->bufSize * slabPool->desiredNumBuffers;
317 struct _WsbmSlabKernelBO *kbo;
318 struct _WsbmSlabKernelBO *kboTmp;
319 int ret;
320
321 /*
322 * FIXME: We should perhaps allow some variation in slabsize in order
323 * to efficiently reuse slabs.
324 */
325
326 size = (size <= (uint32_t) slabPool->maxSlabSize) ? size : (uint32_t) slabPool->maxSlabSize;
327 if (size < header->bufSize)
328 size = header->bufSize;
329 size = (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1);
330 WSBM_MUTEX_LOCK(&cache->mutex);
331
332 kbo = NULL;
333
334 retry:
335 head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ?
336 &cache->cached : &cache->unCached;
337
338 WSBMLISTFOREACH(list, head) {
339 kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head);
340
341 if ((kboTmp->actualSize == size) &&
342 (slabPool->pageAlignment == 0 ||
343 (kboTmp->pageAlignment % slabPool->pageAlignment) == 0)) {
344
345 if (!kbo)
346 kbo = kboTmp;
347
348 if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0)
349 break;
350
351 }
352 }
353
354 if (kbo) {
355 WSBMLISTDELINIT(&kbo->head);
356 WSBMLISTDELINIT(&kbo->timeoutHead);
357 }
358
359 WSBM_MUTEX_UNLOCK(&cache->mutex);
360
361 if (kbo) {
362 uint32_t new_mask =
363 kbo->proposedPlacement ^ slabPool->proposedPlacement;
364
365 ret = 0;
366 if (new_mask) {
367 union ttm_pl_setstatus_arg arg;
368 struct ttm_pl_setstatus_req *req = &arg.req;
369 struct ttm_pl_rep *rep = &arg.rep;
370
371 req->handle = kbo->kBuf.handle;
372 req->set_placement = slabPool->proposedPlacement & new_mask;
373 req->clr_placement = ~slabPool->proposedPlacement & new_mask;
374 DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
375 slabPool->devOffset + TTM_PL_SETSTATUS,
376 arg, ret);
377 if (ret == 0) {
378 kbo->kBuf.gpuOffset = rep->gpu_offset;
379 kbo->kBuf.placement = rep->placement;
380 }
381 kbo->proposedPlacement = slabPool->proposedPlacement;
382 }
383
384 if (ret == 0)
385 return kbo;
386
387 wsbmFreeKernelBO(kbo);
388 kbo = NULL;
389 goto retry;
390 }
391
392 kbo = calloc(1, sizeof(*kbo));
393 if (!kbo)
394 return NULL;
395
396 {
397 union ttm_pl_create_arg arg;
398
399 kbo->slabPool = slabPool;
400 WSBMINITLISTHEAD(&kbo->head);
401 WSBMINITLISTHEAD(&kbo->timeoutHead);
402
403 arg.req.size = size;
404 arg.req.placement = slabPool->proposedPlacement;
405 arg.req.page_alignment = slabPool->pageAlignment;
406
407 DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
408 slabPool->devOffset + TTM_PL_CREATE,
409 arg, ret);
410 if (ret)
411 goto out_err0;
412
413 kbo->kBuf.gpuOffset = arg.rep.gpu_offset;
414 kbo->kBuf.placement = arg.rep.placement;
415 kbo->kBuf.handle = arg.rep.handle;
416
417 kbo->actualSize = arg.rep.bo_size;
418 kbo->mapHandle = arg.rep.map_handle;
419 kbo->proposedPlacement = slabPool->proposedPlacement;
420 }
421
422 kbo->virtual = mmap(0, kbo->actualSize,
423 PROT_READ | PROT_WRITE, MAP_SHARED,
424 slabPool->pool.fd, kbo->mapHandle);
425
426 if (kbo->virtual == MAP_FAILED) {
427 ret = -errno;
428 goto out_err1;
429 }
430
431 return kbo;
432
433 out_err1:
434 {
435 struct ttm_pl_reference_req arg = {.handle = kbo->kBuf.handle };
436
437 (void)drmCommandWrite(slabPool->pool.fd,
438 slabPool->devOffset + TTM_PL_UNREF,
439 &arg, sizeof(arg));
440 }
441 out_err0:
442 free(kbo);
443 return NULL;
444 }
445
446 static int
wsbmAllocSlab(struct _WsbmSlabSizeHeader * header)447 wsbmAllocSlab(struct _WsbmSlabSizeHeader *header)
448 {
449 struct _WsbmSlab *slab;
450 struct _WsbmSlabBuffer *sBuf;
451 uint32_t numBuffers;
452 uint32_t ret;
453 uint32_t i;
454
455 slab = calloc(1, sizeof(*slab));
456 if (!slab)
457 return -ENOMEM;
458
459 slab->kbo = wsbmAllocKernelBO(header);
460 if (!slab->kbo) {
461 ret = -ENOMEM;
462 goto out_err0;
463 }
464
465 numBuffers = slab->kbo->actualSize / header->bufSize;
466 if (!numBuffers) {
467 ret = -ENOMEM;
468 goto out_err1;
469 }
470
471 slab->buffers = calloc(numBuffers, sizeof(*slab->buffers));
472 if (!slab->buffers) {
473 ret = -ENOMEM;
474 goto out_err1;
475 }
476
477 WSBMINITLISTHEAD(&slab->head);
478 WSBMINITLISTHEAD(&slab->freeBuffers);
479 slab->numBuffers = numBuffers;
480 slab->numFree = 0;
481 slab->header = header;
482
483 sBuf = slab->buffers;
484 for (i = 0; i < numBuffers; ++i) {
485 ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool);
486 if (ret)
487 goto out_err2;
488 sBuf->parent = slab;
489 sBuf->start = i * header->bufSize;
490 sBuf->virtual = (void *)((uint8_t *) slab->kbo->virtual +
491 sBuf->start);
492 wsbmAtomicSet(&sBuf->writers, 0);
493 sBuf->isSlabBuffer = 1;
494 WSBM_COND_INIT(&sBuf->event);
495 WSBMLISTADDTAIL(&sBuf->head, &slab->freeBuffers);
496 slab->numFree++;
497 sBuf++;
498 }
499
500 WSBMLISTADDTAIL(&slab->head, &header->slabs);
501
502 return 0;
503
504 out_err2:
505 sBuf = slab->buffers;
506 for (i = 0; i < numBuffers; ++i) {
507 if (sBuf->parent == slab) {
508 WSBM_COND_FREE(&sBuf->event);
509 wsbmBufStorageTakedown(&sBuf->storage);
510 }
511 sBuf++;
512 }
513 free(slab->buffers);
514 out_err1:
515 wsbmSetKernelBOFree(header->slabPool->cache, slab->kbo);
516 out_err0:
517 free(slab);
518 return ret;
519 }
520
521 /*
522 * Delete a buffer from the slab header delayed list and put
523 * it on the slab free list.
524 */
525
526 static void
wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer * buf)527 wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer *buf)
528 {
529 struct _WsbmSlab *slab = buf->parent;
530 struct _WsbmSlabSizeHeader *header = slab->header;
531 struct _WsbmListHead *list = &buf->head;
532
533 WSBMLISTDEL(list);
534 WSBMLISTADDTAIL(list, &slab->freeBuffers);
535 slab->numFree++;
536
537 if (slab->head.next == &slab->head)
538 WSBMLISTADDTAIL(&slab->head, &header->slabs);
539
540 if (slab->numFree == slab->numBuffers) {
541 list = &slab->head;
542 WSBMLISTDEL(list);
543 WSBMLISTADDTAIL(list, &header->freeSlabs);
544 }
545
546 if (header->slabs.next == &header->slabs ||
547 slab->numFree != slab->numBuffers) {
548
549 struct _WsbmListHead *next;
550 struct _WsbmSlabCache *cache = header->slabPool->cache;
551
552 WSBMLISTFOREACHSAFE(list, next, &header->freeSlabs) {
553 uint32_t i;
554 struct _WsbmSlabBuffer *sBuf;
555
556 slab = WSBMLISTENTRY(list, struct _WsbmSlab, head);
557
558 WSBMLISTDELINIT(list);
559
560 sBuf = slab->buffers;
561 for (i = 0; i < slab->numBuffers; ++i) {
562 if (sBuf->parent == slab) {
563 WSBM_COND_FREE(&sBuf->event);
564 wsbmBufStorageTakedown(&sBuf->storage);
565 }
566 sBuf++;
567 }
568 wsbmSetKernelBOFree(cache, slab->kbo);
569 free(slab->buffers);
570 free(slab);
571 }
572 }
573 }
574
575 static void
wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader * header,int wait)576 wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader *header, int wait)
577 {
578 struct _WsbmListHead *list, *prev, *first, *head;
579 struct _WsbmSlabBuffer *sBuf;
580 struct _WsbmSlab *slab;
581 int firstWasSignaled = 1;
582 int signaled;
583 uint32_t i;
584 int ret;
585
586 /*
587 * Rerun the freeing test if the youngest tested buffer
588 * was signaled, since there might be more idle buffers
589 * in the delay list.
590 */
591
592 while (firstWasSignaled) {
593 firstWasSignaled = 0;
594 signaled = 0;
595 first = header->delayedBuffers.next;
596
597 /* Only examine the oldest 1/3 of delayed buffers:
598 */
599 if (header->numDelayed > 3) {
600 for (i = 0; i < header->numDelayed; i += 3) {
601 first = first->next;
602 }
603 }
604
605 /*
606 * No need to take the buffer mutex for each buffer we loop
607 * through since we're currently the only user.
608 */
609
610 head = first->next;
611 WSBMLISTFOREACHPREVSAFE(list, prev, head) {
612
613 if (list == &header->delayedBuffers)
614 break;
615
616 sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
617
618 slab = sBuf->parent;
619
620 if (!signaled) {
621 if (wait) {
622 ret = wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);
623 if (ret)
624 break;
625 signaled = 1;
626 wait = 0;
627 } else {
628 signaled =
629 wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);
630 #ifdef DEBUG_FENCESIGNALED
631 fencesignaled++;
632 #endif
633 }
634 if (signaled) {
635 if (list == first)
636 firstWasSignaled = 1;
637 wsbmFenceUnreference(&sBuf->fence);
638 header->numDelayed--;
639 wsbmSlabFreeBufferLocked(sBuf);
640 } else
641 break;
642 } else if (wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {
643 wsbmFenceUnreference(&sBuf->fence);
644 header->numDelayed--;
645 wsbmSlabFreeBufferLocked(sBuf);
646 }
647 }
648 }
649 }
650
651 static struct _WsbmSlabBuffer *
wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader * header)652 wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header)
653 {
654 static struct _WsbmSlabBuffer *buf;
655 struct _WsbmSlab *slab;
656 struct _WsbmListHead *list;
657 int count = WSBM_SLABPOOL_ALLOC_RETRIES;
658
659 WSBM_MUTEX_LOCK(&header->mutex);
660 while (header->slabs.next == &header->slabs && count > 0) {
661 wsbmSlabCheckFreeLocked(header, 0);
662 if (header->slabs.next != &header->slabs)
663 break;
664
665 WSBM_MUTEX_UNLOCK(&header->mutex);
666 if (count != WSBM_SLABPOOL_ALLOC_RETRIES)
667 usleep(1000);
668 WSBM_MUTEX_LOCK(&header->mutex);
669 (void)wsbmAllocSlab(header);
670 count--;
671 }
672
673 list = header->slabs.next;
674 if (list == &header->slabs) {
675 WSBM_MUTEX_UNLOCK(&header->mutex);
676 return NULL;
677 }
678 slab = WSBMLISTENTRY(list, struct _WsbmSlab, head);
679 if (--slab->numFree == 0)
680 WSBMLISTDELINIT(list);
681
682 list = slab->freeBuffers.next;
683 WSBMLISTDELINIT(list);
684
685 WSBM_MUTEX_UNLOCK(&header->mutex);
686 buf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
687
688 buf->storage.destroyContainer = NULL;
689
690 #ifdef DEBUG_FENCESIGNALED
691 createbuffer++;
692 #endif
693 return buf;
694 }
695
696 static struct _WsbmBufStorage *
pool_create(struct _WsbmBufferPool * pool,unsigned long size,uint32_t placement,unsigned alignment)697 pool_create(struct _WsbmBufferPool *pool, unsigned long size,
698 uint32_t placement, unsigned alignment)
699 {
700 struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool);
701 struct _WsbmSlabSizeHeader *header;
702 struct _WsbmSlabBuffer *sBuf;
703 uint32_t i;
704 int ret;
705
706 /*
707 * FIXME: Check for compatibility.
708 */
709
710 header = slabPool->headers;
711 for (i = 0; i < slabPool->numBuckets; ++i) {
712 if (header->bufSize >= size)
713 break;
714 header++;
715 }
716
717 if (i < slabPool->numBuckets) {
718 sBuf = wsbmSlabAllocBuffer(header);
719 return ((sBuf) ? &sBuf->storage : NULL);
720 }
721
722 /*
723 * Fall back to allocate a buffer object directly from DRM.
724 * and wrap it in a wsbmBO structure.
725 */
726
727 sBuf = calloc(1, sizeof(*sBuf));
728
729 if (!sBuf)
730 return NULL;
731
732 if (alignment) {
733 if ((alignment < slabPool->pageSize)
734 && (slabPool->pageSize % alignment))
735 goto out_err0;
736 if ((alignment > slabPool->pageSize)
737 && (alignment % slabPool->pageSize))
738 goto out_err0;
739 }
740
741 ret = wsbmBufStorageInit(&sBuf->storage, pool);
742 if (ret)
743 goto out_err0;
744
745 ret = WSBM_COND_INIT(&sBuf->event);
746 if (ret)
747 goto out_err1;
748
749 {
750 union ttm_pl_create_arg arg;
751
752 arg.req.size = size;
753 arg.req.placement = placement;
754 arg.req.page_alignment = alignment / slabPool->pageSize;
755
756 DRMRESTARTCOMMANDWRITEREAD(pool->fd,
757 slabPool->devOffset + TTM_PL_CREATE,
758 arg, ret);
759
760 if (ret)
761 goto out_err2;
762
763 sBuf->kBuf.gpuOffset = arg.rep.gpu_offset;
764 sBuf->kBuf.placement = arg.rep.placement;
765 sBuf->kBuf.handle = arg.rep.handle;
766 sBuf->mapHandle = arg.rep.map_handle;
767 sBuf->requestedSize = size;
768
769 sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED,
770 pool->fd, sBuf->mapHandle);
771
772 if (sBuf->virtual == MAP_FAILED)
773 goto out_err3;
774 }
775
776 wsbmAtomicSet(&sBuf->writers, 0);
777 return &sBuf->storage;
778 out_err3:
779 {
780 struct ttm_pl_reference_req arg;
781
782 arg.handle = sBuf->kBuf.handle;
783 (void)drmCommandWriteRead(pool->fd,
784 slabPool->devOffset + TTM_PL_UNREF,
785 &arg, sizeof(arg));
786 }
787 out_err2:
788 WSBM_COND_FREE(&sBuf->event);
789 out_err1:
790 wsbmBufStorageTakedown(&sBuf->storage);
791 out_err0:
792 free(sBuf);
793 return NULL;
794 }
795
796 static void
pool_destroy(struct _WsbmBufStorage ** p_buf)797 pool_destroy(struct _WsbmBufStorage **p_buf)
798 {
799 struct _WsbmBufStorage *buf = *p_buf;
800 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
801 struct _WsbmSlab *slab;
802 struct _WsbmSlabSizeHeader *header;
803
804 *p_buf = NULL;
805
806 if (!sBuf->isSlabBuffer) {
807 struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf);
808 struct ttm_pl_reference_req arg;
809
810 if (sBuf->virtual != NULL) {
811 (void)munmap(sBuf->virtual, sBuf->requestedSize);
812 sBuf->virtual = NULL;
813 }
814
815 arg.handle = sBuf->kBuf.handle;
816 (void)drmCommandWrite(slabPool->pool.fd,
817 slabPool->devOffset + TTM_PL_UNREF,
818 &arg, sizeof(arg));
819
820 WSBM_COND_FREE(&sBuf->event);
821 wsbmBufStorageTakedown(&sBuf->storage);
822 free(sBuf);
823 return;
824 }
825
826 slab = sBuf->parent;
827 header = slab->header;
828
829 /*
830 * No need to take the buffer mutex below since we're the only user.
831 */
832
833 WSBM_MUTEX_LOCK(&header->mutex);
834 sBuf->unFenced = 0;
835 wsbmAtomicSet(&sBuf->writers, 0);
836 wsbmAtomicSet(&sBuf->storage.refCount, 1);
837
838 if (sBuf->fence && !wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {
839 WSBMLISTADDTAIL(&sBuf->head, &header->delayedBuffers);
840 header->numDelayed++;
841 } else {
842 if (sBuf->fence)
843 wsbmFenceUnreference(&sBuf->fence);
844 wsbmSlabFreeBufferLocked(sBuf);
845 }
846
847 WSBM_MUTEX_UNLOCK(&header->mutex);
848 }
849
850 static void
waitIdleLocked(struct _WsbmSlabBuffer * sBuf,int lazy)851 waitIdleLocked(struct _WsbmSlabBuffer *sBuf, int lazy)
852 {
853 struct _WsbmBufStorage *storage = &sBuf->storage;
854
855 while (sBuf->unFenced || sBuf->fence != NULL) {
856
857 if (sBuf->unFenced)
858 WSBM_COND_WAIT(&sBuf->event, &storage->mutex);
859
860 if (sBuf->fence != NULL) {
861 if (!wsbmFenceSignaled(sBuf->fence, sBuf->fenceType)) {
862 struct _WsbmFenceObject *fence =
863 wsbmFenceReference(sBuf->fence);
864
865 WSBM_MUTEX_UNLOCK(&storage->mutex);
866 (void)wsbmFenceFinish(fence, sBuf->fenceType, lazy);
867 WSBM_MUTEX_LOCK(&storage->mutex);
868 if (sBuf->fence == fence)
869 wsbmFenceUnreference(&sBuf->fence);
870
871 wsbmFenceUnreference(&fence);
872 } else {
873 wsbmFenceUnreference(&sBuf->fence);
874 }
875 }
876 }
877 }
878
879 static int
pool_waitIdle(struct _WsbmBufStorage * buf,int lazy)880 pool_waitIdle(struct _WsbmBufStorage *buf, int lazy)
881 {
882 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
883
884 WSBM_MUTEX_LOCK(&buf->mutex);
885 waitIdleLocked(sBuf, lazy);
886 WSBM_MUTEX_UNLOCK(&buf->mutex);
887
888 return 0;
889 }
890
891 static int
pool_map(struct _WsbmBufStorage * buf,unsigned mode,void ** virtual)892 pool_map(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)), void **virtual)
893 {
894 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
895
896 *virtual = sBuf->virtual;
897
898 return 0;
899 }
900
901 static void
pool_releaseFromCpu(struct _WsbmBufStorage * buf,unsigned mode)902 pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)))
903 {
904 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
905
906 if (wsbmAtomicDecZero(&sBuf->writers))
907 WSBM_COND_BROADCAST(&sBuf->event);
908 }
909
910 static int
pool_syncForCpu(struct _WsbmBufStorage * buf,unsigned mode)911 pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode)
912 {
913 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
914 int ret = 0;
915
916 WSBM_MUTEX_LOCK(&buf->mutex);
917 if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) {
918 int signaled;
919
920 if (sBuf->unFenced) {
921 ret = -EBUSY;
922 goto out_unlock;
923 }
924
925 if (sBuf->isSlabBuffer)
926 signaled = (sBuf->fence == NULL) ||
927 wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType);
928 else
929 signaled = (sBuf->fence == NULL) ||
930 wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);
931
932 ret = 0;
933 if (signaled) {
934 wsbmFenceUnreference(&sBuf->fence);
935 wsbmAtomicInc(&sBuf->writers);
936 } else
937 ret = -EBUSY;
938 goto out_unlock;
939 }
940 waitIdleLocked(sBuf, 0);
941 wsbmAtomicInc(&sBuf->writers);
942 out_unlock:
943 WSBM_MUTEX_UNLOCK(&buf->mutex);
944 return ret;
945 }
946
947 static void
pool_unmap(struct _WsbmBufStorage * buf)948 pool_unmap(struct _WsbmBufStorage *buf __attribute__ ((unused)))
949 {
950 ;
951 }
952
953 static unsigned long
pool_poolOffset(struct _WsbmBufStorage * buf)954 pool_poolOffset(struct _WsbmBufStorage *buf)
955 {
956 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
957
958 return sBuf->start;
959 }
960
961 static unsigned long
pool_size(struct _WsbmBufStorage * buf)962 pool_size(struct _WsbmBufStorage *buf)
963 {
964 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
965
966 if (!sBuf->isSlabBuffer)
967 return sBuf->requestedSize;
968
969 return sBuf->parent->header->bufSize;
970 }
971
972 static struct _WsbmKernelBuf *
pool_kernel(struct _WsbmBufStorage * buf)973 pool_kernel(struct _WsbmBufStorage *buf)
974 {
975 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
976
977 return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf;
978 }
979
980 static unsigned long
pool_offset(struct _WsbmBufStorage * buf)981 pool_offset(struct _WsbmBufStorage *buf)
982 {
983 return pool_kernel(buf)->gpuOffset + pool_poolOffset(buf);
984 }
985
986 static void
pool_fence(struct _WsbmBufStorage * buf,struct _WsbmFenceObject * fence)987 pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence)
988 {
989 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
990 struct _WsbmKernelBuf *kBuf;
991
992 WSBM_MUTEX_LOCK(&buf->mutex);
993 if (sBuf->fence)
994 wsbmFenceUnreference(&sBuf->fence);
995
996 kBuf = pool_kernel(buf);
997 sBuf->fenceType = kBuf->fence_type_mask;
998 if (!wsbmFenceSignaledCached(fence, sBuf->fenceType))
999 sBuf->fence = wsbmFenceReference(fence);
1000
1001 sBuf->unFenced = 0;
1002 WSBM_COND_BROADCAST(&sBuf->event);
1003 WSBM_MUTEX_UNLOCK(&buf->mutex);
1004 }
1005
1006 static int
pool_validate(struct _WsbmBufStorage * buf,uint64_t set_flags,uint64_t clr_flags)1007 pool_validate(struct _WsbmBufStorage *buf,
1008 uint64_t set_flags __attribute__ ((unused)), uint64_t clr_flags __attribute__ ((unused)))
1009 {
1010 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
1011
1012 WSBM_MUTEX_LOCK(&buf->mutex);
1013 while (wsbmAtomicRead(&sBuf->writers) != 0) {
1014 WSBM_COND_WAIT(&sBuf->event, &buf->mutex);
1015 }
1016
1017 sBuf->unFenced = 1;
1018 WSBM_MUTEX_UNLOCK(&buf->mutex);
1019 return 0;
1020 }
1021
1022 static void
pool_unvalidate(struct _WsbmBufStorage * buf)1023 pool_unvalidate(struct _WsbmBufStorage *buf)
1024 {
1025 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
1026
1027 WSBM_MUTEX_LOCK(&buf->mutex);
1028 if (sBuf->unFenced) {
1029 sBuf->unFenced = 0;
1030 WSBM_COND_BROADCAST(&sBuf->event);
1031 }
1032 WSBM_MUTEX_UNLOCK(&buf->mutex);
1033 }
1034
1035 struct _WsbmSlabCache *
wsbmSlabCacheInit(uint32_t checkIntervalMsec,uint32_t slabTimeoutMsec)1036 wsbmSlabCacheInit(uint32_t checkIntervalMsec, uint32_t slabTimeoutMsec)
1037 {
1038 struct _WsbmSlabCache *tmp;
1039
1040 tmp = calloc(1, sizeof(*tmp));
1041 if (!tmp)
1042 return NULL;
1043
1044 WSBM_MUTEX_INIT(&tmp->mutex);
1045 WSBM_MUTEX_LOCK(&tmp->mutex);
1046 tmp->slabTimeout.tv_usec = slabTimeoutMsec * 1000;
1047 tmp->slabTimeout.tv_sec = tmp->slabTimeout.tv_usec / 1000000;
1048 tmp->slabTimeout.tv_usec -= tmp->slabTimeout.tv_sec * 1000000;
1049
1050 tmp->checkInterval.tv_usec = checkIntervalMsec * 1000;
1051 tmp->checkInterval.tv_sec = tmp->checkInterval.tv_usec / 1000000;
1052 tmp->checkInterval.tv_usec -= tmp->checkInterval.tv_sec * 1000000;
1053
1054 gettimeofday(&tmp->nextCheck, NULL);
1055 wsbmTimeAdd(&tmp->nextCheck, &tmp->checkInterval);
1056 WSBMINITLISTHEAD(&tmp->timeoutList);
1057 WSBMINITLISTHEAD(&tmp->unCached);
1058 WSBMINITLISTHEAD(&tmp->cached);
1059 WSBM_MUTEX_UNLOCK(&tmp->mutex);
1060
1061 return tmp;
1062 }
1063
1064 void
wsbmSlabCacheFinish(struct _WsbmSlabCache * cache)1065 wsbmSlabCacheFinish(struct _WsbmSlabCache *cache)
1066 {
1067 struct timeval time;
1068
1069 time = cache->nextCheck;
1070 WSBM_MUTEX_LOCK(&cache->mutex);
1071 wsbmTimeAdd(&time, &cache->checkInterval);
1072 wsbmFreeTimeoutKBOsLocked(cache, &time);
1073 WSBM_MUTEX_UNLOCK(&cache->mutex);
1074
1075 assert(cache->timeoutList.next == &cache->timeoutList);
1076 assert(cache->unCached.next == &cache->unCached);
1077 assert(cache->cached.next == &cache->cached);
1078
1079 WSBM_MUTEX_FREE(&cache->mutex);
1080 free(cache);
1081 }
1082
1083 static void
wsbmInitSizeHeader(struct _WsbmSlabPool * slabPool,uint32_t size,struct _WsbmSlabSizeHeader * header)1084 wsbmInitSizeHeader(struct _WsbmSlabPool *slabPool, uint32_t size,
1085 struct _WsbmSlabSizeHeader *header)
1086 {
1087 WSBM_MUTEX_INIT(&header->mutex);
1088 WSBM_MUTEX_LOCK(&header->mutex);
1089
1090 WSBMINITLISTHEAD(&header->slabs);
1091 WSBMINITLISTHEAD(&header->freeSlabs);
1092 WSBMINITLISTHEAD(&header->delayedBuffers);
1093
1094 header->numDelayed = 0;
1095 header->slabPool = slabPool;
1096 header->bufSize = size;
1097
1098 WSBM_MUTEX_UNLOCK(&header->mutex);
1099 }
1100
1101 static void
wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader * header)1102 wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader *header)
1103 {
1104 struct _WsbmListHead *list, *next;
1105 struct _WsbmSlabBuffer *sBuf;
1106
1107 WSBM_MUTEX_LOCK(&header->mutex);
1108 WSBMLISTFOREACHSAFE(list, next, &header->delayedBuffers) {
1109 sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
1110
1111 if (sBuf->fence) {
1112 (void)wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);
1113 wsbmFenceUnreference(&sBuf->fence);
1114 }
1115 header->numDelayed--;
1116 wsbmSlabFreeBufferLocked(sBuf);
1117 }
1118 WSBM_MUTEX_UNLOCK(&header->mutex);
1119 WSBM_MUTEX_FREE(&header->mutex);
1120 }
1121
1122 static void
pool_takedown(struct _WsbmBufferPool * pool)1123 pool_takedown(struct _WsbmBufferPool *pool)
1124 {
1125 struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool);
1126 unsigned int i;
1127
1128 for (i = 0; i < slabPool->numBuckets; ++i) {
1129 wsbmFinishSizeHeader(&slabPool->headers[i]);
1130 }
1131
1132 free(slabPool->headers);
1133 free(slabPool->bucketSizes);
1134 free(slabPool);
1135 }
1136
1137 struct _WsbmBufferPool *
wsbmSlabPoolInit(int fd,uint32_t devOffset,uint32_t placement,uint32_t validMask,uint32_t smallestSize,uint32_t numSizes,uint32_t desiredNumBuffers,uint32_t maxSlabSize,uint32_t pageAlignment,struct _WsbmSlabCache * cache)1138 wsbmSlabPoolInit(int fd,
1139 uint32_t devOffset,
1140 uint32_t placement,
1141 uint32_t validMask,
1142 uint32_t smallestSize,
1143 uint32_t numSizes,
1144 uint32_t desiredNumBuffers,
1145 uint32_t maxSlabSize,
1146 uint32_t pageAlignment, struct _WsbmSlabCache *cache)
1147 {
1148 struct _WsbmBufferPool *pool;
1149 struct _WsbmSlabPool *slabPool;
1150 uint32_t i;
1151
1152 slabPool = calloc(1, sizeof(*slabPool));
1153 if (!slabPool)
1154 return NULL;
1155
1156 pool = &slabPool->pool;
1157
1158 slabPool->bucketSizes = calloc(numSizes, sizeof(*slabPool->bucketSizes));
1159 if (!slabPool->bucketSizes)
1160 goto out_err0;
1161
1162 slabPool->headers = calloc(numSizes, sizeof(*slabPool->headers));
1163 if (!slabPool->headers)
1164 goto out_err1;
1165
1166 slabPool->devOffset = devOffset;
1167 slabPool->cache = cache;
1168 slabPool->proposedPlacement = placement;
1169 slabPool->validMask = validMask;
1170 slabPool->numBuckets = numSizes;
1171 slabPool->pageSize = getpagesize();
1172 slabPool->pageAlignment = pageAlignment;
1173 slabPool->maxSlabSize = maxSlabSize;
1174 slabPool->desiredNumBuffers = desiredNumBuffers;
1175
1176 for (i = 0; i < slabPool->numBuckets; ++i) {
1177 slabPool->bucketSizes[i] = (smallestSize << i);
1178 wsbmInitSizeHeader(slabPool, slabPool->bucketSizes[i],
1179 &slabPool->headers[i]);
1180 }
1181
1182 pool->fd = fd;
1183 pool->map = &pool_map;
1184 pool->unmap = &pool_unmap;
1185 pool->destroy = &pool_destroy;
1186 pool->offset = &pool_offset;
1187 pool->poolOffset = &pool_poolOffset;
1188 pool->size = &pool_size;
1189 pool->create = &pool_create;
1190 pool->fence = &pool_fence;
1191 pool->kernel = &pool_kernel;
1192 pool->validate = &pool_validate;
1193 pool->unvalidate = &pool_unvalidate;
1194 pool->waitIdle = &pool_waitIdle;
1195 pool->takeDown = &pool_takedown;
1196 pool->releasefromcpu = &pool_releaseFromCpu;
1197 pool->syncforcpu = &pool_syncForCpu;
1198
1199 return pool;
1200
1201 out_err1:
1202 free(slabPool->bucketSizes);
1203 out_err0:
1204 free(slabPool);
1205
1206 return NULL;
1207 }
1208