198 slabPoolFromBuf(struct _WsbmSlabBuffer *sBuf)  in slabPoolFromBuf()  argument
200     return slabPoolFromPool(sBuf->storage.pool);  in slabPoolFromBuf()
450     struct _WsbmSlabBuffer *sBuf;  in wsbmAllocSlab()  local
479     sBuf = slab->buffers;  in wsbmAllocSlab()
481 	ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool);  in wsbmAllocSlab()
484 	sBuf->parent = slab;  in wsbmAllocSlab()
485 	sBuf->start = i * header->bufSize;  in wsbmAllocSlab()
486 	sBuf->virtual = (void *)((uint8_t *) slab->kbo->virtual +  in wsbmAllocSlab()
487 				 sBuf->start);  in wsbmAllocSlab()
488 	wsbmAtomicSet(&sBuf->writers, 0);  in wsbmAllocSlab()
489 	sBuf->isSlabBuffer = 1;  in wsbmAllocSlab()
490 	WSBM_COND_INIT(&sBuf->event);  in wsbmAllocSlab()
491 	WSBMLISTADDTAIL(&sBuf->head, &slab->freeBuffers);  in wsbmAllocSlab()
493 	sBuf++;  in wsbmAllocSlab()
501     sBuf = slab->buffers;  in wsbmAllocSlab()
503 	if (sBuf->parent == slab) {  in wsbmAllocSlab()
504 	    WSBM_COND_FREE(&sBuf->event);  in wsbmAllocSlab()
505 	    wsbmBufStorageTakedown(&sBuf->storage);  in wsbmAllocSlab()
507 	sBuf++;  in wsbmAllocSlab()
550 	    struct _WsbmSlabBuffer *sBuf;  in wsbmSlabFreeBufferLocked()  local
556 	    sBuf = slab->buffers;  in wsbmSlabFreeBufferLocked()
558 		if (sBuf->parent == slab) {  in wsbmSlabFreeBufferLocked()
559 		    WSBM_COND_FREE(&sBuf->event);  in wsbmSlabFreeBufferLocked()
560 		    wsbmBufStorageTakedown(&sBuf->storage);  in wsbmSlabFreeBufferLocked()
562 		sBuf++;  in wsbmSlabFreeBufferLocked()
575     struct _WsbmSlabBuffer *sBuf;  in wsbmSlabCheckFreeLocked()  local
612 	    sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);  in wsbmSlabCheckFreeLocked()
614 	    slab = sBuf->parent;  in wsbmSlabCheckFreeLocked()
618 		    ret = wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);  in wsbmSlabCheckFreeLocked()
625 			wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);  in wsbmSlabCheckFreeLocked()
633 		    wsbmFenceUnreference(&sBuf->fence);  in wsbmSlabCheckFreeLocked()
635 		    wsbmSlabFreeBufferLocked(sBuf);  in wsbmSlabCheckFreeLocked()
638 	    } else if (wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {  in wsbmSlabCheckFreeLocked()
639 		wsbmFenceUnreference(&sBuf->fence);  in wsbmSlabCheckFreeLocked()
641 		wsbmSlabFreeBufferLocked(sBuf);  in wsbmSlabCheckFreeLocked()
698     struct _WsbmSlabBuffer *sBuf;  in pool_create()  local
714 	sBuf = wsbmSlabAllocBuffer(header);  in pool_create()
715 	return ((sBuf) ? &sBuf->storage : NULL);  in pool_create()
723     sBuf = calloc(1, sizeof(*sBuf));  in pool_create()
725     if (!sBuf)  in pool_create()
737     ret = wsbmBufStorageInit(&sBuf->storage, pool);  in pool_create()
741     ret = WSBM_COND_INIT(&sBuf->event);  in pool_create()
759 	sBuf->kBuf.gpuOffset = arg.rep.gpu_offset;  in pool_create()
760 	sBuf->kBuf.placement = arg.rep.placement;  in pool_create()
761 	sBuf->kBuf.handle = arg.rep.handle;  in pool_create()
762 	sBuf->mapHandle = arg.rep.map_handle;  in pool_create()
763 	sBuf->requestedSize = size;  in pool_create()
765 	sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED,  in pool_create()
766 			     pool->fd, sBuf->mapHandle);  in pool_create()
768 	if (sBuf->virtual == MAP_FAILED)  in pool_create()
772     wsbmAtomicSet(&sBuf->writers, 0);  in pool_create()
773     return &sBuf->storage;  in pool_create()
778 	arg.handle = sBuf->kBuf.handle;  in pool_create()
784     WSBM_COND_FREE(&sBuf->event);  in pool_create()
786     wsbmBufStorageTakedown(&sBuf->storage);  in pool_create()
788     free(sBuf);  in pool_create()
796     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);  in pool_destroy()  local
802     if (!sBuf->isSlabBuffer) {  in pool_destroy()
803 	struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf);  in pool_destroy()
806 	if (sBuf->virtual != NULL) {  in pool_destroy()
807 	    (void)munmap(sBuf->virtual, sBuf->requestedSize);  in pool_destroy()
808 	    sBuf->virtual = NULL;  in pool_destroy()
811 	arg.handle = sBuf->kBuf.handle;  in pool_destroy()
816 	WSBM_COND_FREE(&sBuf->event);  in pool_destroy()
817 	wsbmBufStorageTakedown(&sBuf->storage);  in pool_destroy()
818 	free(sBuf);  in pool_destroy()
822     slab = sBuf->parent;  in pool_destroy()
830     sBuf->unFenced = 0;  in pool_destroy()
831     wsbmAtomicSet(&sBuf->writers, 0);  in pool_destroy()
832     wsbmAtomicSet(&sBuf->storage.refCount, 1);  in pool_destroy()
834     if (sBuf->fence && !wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {  in pool_destroy()
835 	WSBMLISTADDTAIL(&sBuf->head, &header->delayedBuffers);  in pool_destroy()
838 	if (sBuf->fence)  in pool_destroy()
839 	    wsbmFenceUnreference(&sBuf->fence);  in pool_destroy()
840 	wsbmSlabFreeBufferLocked(sBuf);  in pool_destroy()
847 waitIdleLocked(struct _WsbmSlabBuffer *sBuf, int lazy)  in waitIdleLocked()  argument
849     struct _WsbmBufStorage *storage = &sBuf->storage;  in waitIdleLocked()
851     while (sBuf->unFenced || sBuf->fence != NULL) {  in waitIdleLocked()
853 	if (sBuf->unFenced)  in waitIdleLocked()
854 	    WSBM_COND_WAIT(&sBuf->event, &storage->mutex);  in waitIdleLocked()
856 	if (sBuf->fence != NULL) {  in waitIdleLocked()
857 	    if (!wsbmFenceSignaled(sBuf->fence, sBuf->fenceType)) {  in waitIdleLocked()
859 		    wsbmFenceReference(sBuf->fence);  in waitIdleLocked()
862 		(void)wsbmFenceFinish(fence, sBuf->fenceType, lazy);  in waitIdleLocked()
864 		if (sBuf->fence == fence)  in waitIdleLocked()
865 		    wsbmFenceUnreference(&sBuf->fence);  in waitIdleLocked()
869 		wsbmFenceUnreference(&sBuf->fence);  in waitIdleLocked()
878     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);  in pool_waitIdle()  local
881     waitIdleLocked(sBuf, lazy);  in pool_waitIdle()
890     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);  in pool_map()  local
892     *virtual = sBuf->virtual;  in pool_map()
900     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);  in pool_releaseFromCpu()  local
902     if (wsbmAtomicDecZero(&sBuf->writers))  in pool_releaseFromCpu()
903 	WSBM_COND_BROADCAST(&sBuf->event);  in pool_releaseFromCpu()
909     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);  in pool_syncForCpu()  local
916 	if (sBuf->unFenced) {  in pool_syncForCpu()
921 	if (sBuf->isSlabBuffer)  in pool_syncForCpu()
922 	    signaled = (sBuf->fence == NULL) ||  in pool_syncForCpu()
923 		wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType);  in pool_syncForCpu()
925 	    signaled = (sBuf->fence == NULL) ||  in pool_syncForCpu()
926 		wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);  in pool_syncForCpu()
930 	    wsbmFenceUnreference(&sBuf->fence);  in pool_syncForCpu()
931 	    wsbmAtomicInc(&sBuf->writers);  in pool_syncForCpu()
936     waitIdleLocked(sBuf, 0);  in pool_syncForCpu()
937     wsbmAtomicInc(&sBuf->writers);  in pool_syncForCpu()
952     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);  in pool_poolOffset()  local
954     return sBuf->start;  in pool_poolOffset()
960     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);  in pool_size()  local
962     if (!sBuf->isSlabBuffer)  in pool_size()
963 	return sBuf->requestedSize;  in pool_size()
965     return sBuf->parent->header->bufSize;  in pool_size()
971     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);  in pool_kernel()  local
973     return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf;  in pool_kernel()
985     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);  in pool_fence()  local
989     if (sBuf->fence)  in pool_fence()
990 	wsbmFenceUnreference(&sBuf->fence);  in pool_fence()
993     sBuf->fenceType = kBuf->fence_type_mask;  in pool_fence()
994     if (!wsbmFenceSignaledCached(fence, sBuf->fenceType))  in pool_fence()
995 	sBuf->fence = wsbmFenceReference(fence);  in pool_fence()
997     sBuf->unFenced = 0;  in pool_fence()
998     WSBM_COND_BROADCAST(&sBuf->event);  in pool_fence()
1006     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);  in pool_validate()  local
1009     while (wsbmAtomicRead(&sBuf->writers) != 0) {  in pool_validate()
1010 	WSBM_COND_WAIT(&sBuf->event, &buf->mutex);  in pool_validate()
1013     sBuf->unFenced = 1;  in pool_validate()
1021     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);  in pool_unvalidate()  local
1024     if (sBuf->unFenced) {  in pool_unvalidate()
1025 	sBuf->unFenced = 0;  in pool_unvalidate()
1026 	WSBM_COND_BROADCAST(&sBuf->event);  in pool_unvalidate()
1101     struct _WsbmSlabBuffer *sBuf;  in wsbmFinishSizeHeader()  local
1105 	sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);  in wsbmFinishSizeHeader()
1107 	if (sBuf->fence) {  in wsbmFinishSizeHeader()
1108 	    (void)wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);  in wsbmFinishSizeHeader()
1109 	    wsbmFenceUnreference(&sBuf->fence);  in wsbmFinishSizeHeader()
1112 	wsbmSlabFreeBufferLocked(sBuf);  in wsbmFinishSizeHeader()