/************************************************************************** * * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA * All Rights Reserved. * Copyright 2009 VMware, Inc., Palo Alto, CA., USA * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * **************************************************************************/ /* * Authors: Thomas Hellstrom */ #ifdef HAVE_CONFIG_H #include "config.h" #endif #include #include #include #include #include #include #include #include #include #include "wsbm_pool.h" #include "wsbm_fencemgr.h" #include "wsbm_priv.h" #include "wsbm_manager.h" #define WSBM_SLABPOOL_ALLOC_RETRIES 100 #define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \ do { \ (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \ } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ #define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \ do { \ (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \ } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ #ifdef DEBUG_FENCESIGNALED static int createbuffer = 0; static int fencesignaled = 0; #endif struct _WsbmSlab; struct _WsbmSlabBuffer { struct _WsbmKernelBuf kBuf; struct _WsbmBufStorage storage; struct _WsbmCond event; /* * Remains constant after creation. */ int isSlabBuffer; struct _WsbmSlab *parent; uint32_t start; void *virtual; unsigned long requestedSize; uint64_t mapHandle; /* * Protected by struct _WsbmSlabSizeHeader::mutex */ struct _WsbmListHead head; /* * Protected by this::mutex */ struct _WsbmFenceObject *fence; uint32_t fenceType; struct _WsbmAtomic writers; /* (Only upping) */ int unFenced; }; struct _WsbmSlabPool; struct _WsbmSlabKernelBO { /* * Constant at creation */ struct _WsbmKernelBuf kBuf; uint32_t pageAlignment; void *virtual; unsigned long actualSize; uint64_t mapHandle; /* * Protected by struct _WsbmSlabCache::mutex */ struct _WsbmSlabPool *slabPool; uint32_t proposedPlacement; struct _WsbmListHead timeoutHead; struct _WsbmListHead head; struct timeval timeFreed; }; struct _WsbmSlab { struct _WsbmListHead head; struct _WsbmListHead freeBuffers; uint32_t numBuffers; uint32_t numFree; struct _WsbmSlabBuffer *buffers; struct _WsbmSlabSizeHeader *header; struct _WsbmSlabKernelBO *kbo; }; struct _WsbmSlabSizeHeader { /* * Constant at creation. */ struct _WsbmSlabPool *slabPool; uint32_t bufSize; /* * Protected by this::mutex */ struct _WsbmListHead slabs; struct _WsbmListHead freeSlabs; struct _WsbmListHead delayedBuffers; uint32_t numDelayed; struct _WsbmMutex mutex; }; struct _WsbmSlabCache { struct timeval slabTimeout; struct timeval checkInterval; struct timeval nextCheck; struct _WsbmListHead timeoutList; struct _WsbmListHead unCached; struct _WsbmListHead cached; struct _WsbmMutex mutex; }; struct _WsbmSlabPool { struct _WsbmBufferPool pool; /* * The data of this structure remains constant after * initialization and thus needs no mutex protection. */ unsigned int devOffset; struct _WsbmSlabCache *cache; uint32_t proposedPlacement; uint32_t validMask; uint32_t *bucketSizes; uint32_t numBuckets; uint32_t pageSize; int pageAlignment; int maxSlabSize; int desiredNumBuffers; struct _WsbmSlabSizeHeader *headers; }; static inline struct _WsbmSlabPool * slabPoolFromPool(struct _WsbmBufferPool *pool) { return containerOf(pool, struct _WsbmSlabPool, pool); } static inline struct _WsbmSlabPool * slabPoolFromBuf(struct _WsbmSlabBuffer *sBuf) { return slabPoolFromPool(sBuf->storage.pool); } static inline struct _WsbmSlabBuffer * slabBuffer(struct _WsbmBufStorage *buf) { return containerOf(buf, struct _WsbmSlabBuffer, storage); } /* * FIXME: Perhaps arrange timeout slabs in size buckets for fast * retreival?? */ static inline int wsbmTimeAfterEq(struct timeval *arg1, struct timeval *arg2) { return ((arg1->tv_sec > arg2->tv_sec) || ((arg1->tv_sec == arg2->tv_sec) && (arg1->tv_usec > arg2->tv_usec))); } static inline void wsbmTimeAdd(struct timeval *arg, struct timeval *add) { unsigned int sec; arg->tv_sec += add->tv_sec; arg->tv_usec += add->tv_usec; sec = arg->tv_usec / 1000000; arg->tv_sec += sec; arg->tv_usec -= sec * 1000000; } static void wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo) { struct ttm_pl_reference_req arg; struct _WsbmSlabPool *slabPool; if (!kbo) return; slabPool = kbo->slabPool; arg.handle = kbo->kBuf.handle; (void)munmap(kbo->virtual, kbo->actualSize); (void)drmCommandWrite(slabPool->pool.fd, slabPool->devOffset + TTM_PL_UNREF, &arg, sizeof(arg)); free(kbo); } static void wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache *cache, struct timeval *time) { struct _WsbmListHead *list, *next; struct _WsbmSlabKernelBO *kbo; if (!wsbmTimeAfterEq(time, &cache->nextCheck)) return; WSBMLISTFOREACHSAFE(list, next, &cache->timeoutList) { kbo = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, timeoutHead); if (!wsbmTimeAfterEq(time, &kbo->timeFreed)) break; WSBMLISTDELINIT(&kbo->timeoutHead); WSBMLISTDELINIT(&kbo->head); wsbmFreeKernelBO(kbo); } cache->nextCheck = *time; wsbmTimeAdd(&cache->nextCheck, &cache->checkInterval); } /* * Add a _SlabKernelBO to the free slab manager. * This means that it is available for reuse, but if it's not * reused in a while, it will be freed. */ static void wsbmSetKernelBOFree(struct _WsbmSlabCache *cache, struct _WsbmSlabKernelBO *kbo) { struct timeval time; struct timeval timeFreed; gettimeofday(&time, NULL); timeFreed = time; WSBM_MUTEX_LOCK(&cache->mutex); wsbmTimeAdd(&timeFreed, &cache->slabTimeout); kbo->timeFreed = timeFreed; if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED) WSBMLISTADD(&kbo->head, &cache->cached); else WSBMLISTADD(&kbo->head, &cache->unCached); WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList); wsbmFreeTimeoutKBOsLocked(cache, &time); WSBM_MUTEX_UNLOCK(&cache->mutex); } /* * Get a _SlabKernelBO for us to use as storage for a slab. */ static struct _WsbmSlabKernelBO * wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header) { struct _WsbmSlabPool *slabPool = header->slabPool; struct _WsbmSlabCache *cache = slabPool->cache; struct _WsbmListHead *list, *head; uint32_t size = header->bufSize * slabPool->desiredNumBuffers; struct _WsbmSlabKernelBO *kbo; struct _WsbmSlabKernelBO *kboTmp; int ret; /* * FIXME: We should perhaps allow some variation in slabsize in order * to efficiently reuse slabs. */ size = (size <= (uint32_t) slabPool->maxSlabSize) ? size : (uint32_t) slabPool->maxSlabSize; if (size < header->bufSize) size = header->bufSize; size = (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1); WSBM_MUTEX_LOCK(&cache->mutex); kbo = NULL; retry: head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ? &cache->cached : &cache->unCached; WSBMLISTFOREACH(list, head) { kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head); if ((kboTmp->actualSize == size) && (slabPool->pageAlignment == 0 || (kboTmp->pageAlignment % slabPool->pageAlignment) == 0)) { if (!kbo) kbo = kboTmp; if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0) break; } } if (kbo) { WSBMLISTDELINIT(&kbo->head); WSBMLISTDELINIT(&kbo->timeoutHead); } WSBM_MUTEX_UNLOCK(&cache->mutex); if (kbo) { uint32_t new_mask = kbo->proposedPlacement ^ slabPool->proposedPlacement; ret = 0; if (new_mask) { union ttm_pl_setstatus_arg arg; struct ttm_pl_setstatus_req *req = &arg.req; struct ttm_pl_rep *rep = &arg.rep; req->handle = kbo->kBuf.handle; req->set_placement = slabPool->proposedPlacement & new_mask; req->clr_placement = ~slabPool->proposedPlacement & new_mask; DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, slabPool->devOffset + TTM_PL_SETSTATUS, arg, ret); if (ret == 0) { kbo->kBuf.gpuOffset = rep->gpu_offset; kbo->kBuf.placement = rep->placement; } kbo->proposedPlacement = slabPool->proposedPlacement; } if (ret == 0) return kbo; wsbmFreeKernelBO(kbo); kbo = NULL; goto retry; } kbo = calloc(1, sizeof(*kbo)); if (!kbo) return NULL; { union ttm_pl_create_arg arg; kbo->slabPool = slabPool; WSBMINITLISTHEAD(&kbo->head); WSBMINITLISTHEAD(&kbo->timeoutHead); arg.req.size = size; arg.req.placement = slabPool->proposedPlacement; arg.req.page_alignment = slabPool->pageAlignment; DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, slabPool->devOffset + TTM_PL_CREATE, arg, ret); if (ret) goto out_err0; kbo->kBuf.gpuOffset = arg.rep.gpu_offset; kbo->kBuf.placement = arg.rep.placement; kbo->kBuf.handle = arg.rep.handle; kbo->actualSize = arg.rep.bo_size; kbo->mapHandle = arg.rep.map_handle; kbo->proposedPlacement = slabPool->proposedPlacement; } kbo->virtual = mmap(0, kbo->actualSize, PROT_READ | PROT_WRITE, MAP_SHARED, slabPool->pool.fd, kbo->mapHandle); if (kbo->virtual == MAP_FAILED) { ret = -errno; goto out_err1; } return kbo; out_err1: { struct ttm_pl_reference_req arg = {.handle = kbo->kBuf.handle }; (void)drmCommandWrite(slabPool->pool.fd, slabPool->devOffset + TTM_PL_UNREF, &arg, sizeof(arg)); } out_err0: free(kbo); return NULL; } static int wsbmAllocSlab(struct _WsbmSlabSizeHeader *header) { struct _WsbmSlab *slab; struct _WsbmSlabBuffer *sBuf; uint32_t numBuffers; uint32_t ret; uint32_t i; slab = calloc(1, sizeof(*slab)); if (!slab) return -ENOMEM; slab->kbo = wsbmAllocKernelBO(header); if (!slab->kbo) { ret = -ENOMEM; goto out_err0; } numBuffers = slab->kbo->actualSize / header->bufSize; if (!numBuffers) { ret = -ENOMEM; goto out_err1; } slab->buffers = calloc(numBuffers, sizeof(*slab->buffers)); if (!slab->buffers) { ret = -ENOMEM; goto out_err1; } WSBMINITLISTHEAD(&slab->head); WSBMINITLISTHEAD(&slab->freeBuffers); slab->numBuffers = numBuffers; slab->numFree = 0; slab->header = header; sBuf = slab->buffers; for (i = 0; i < numBuffers; ++i) { ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool); if (ret) goto out_err2; sBuf->parent = slab; sBuf->start = i * header->bufSize; sBuf->virtual = (void *)((uint8_t *) slab->kbo->virtual + sBuf->start); wsbmAtomicSet(&sBuf->writers, 0); sBuf->isSlabBuffer = 1; WSBM_COND_INIT(&sBuf->event); WSBMLISTADDTAIL(&sBuf->head, &slab->freeBuffers); slab->numFree++; sBuf++; } WSBMLISTADDTAIL(&slab->head, &header->slabs); return 0; out_err2: sBuf = slab->buffers; for (i = 0; i < numBuffers; ++i) { if (sBuf->parent == slab) { WSBM_COND_FREE(&sBuf->event); wsbmBufStorageTakedown(&sBuf->storage); } sBuf++; } free(slab->buffers); out_err1: wsbmSetKernelBOFree(header->slabPool->cache, slab->kbo); out_err0: free(slab); return ret; } /* * Delete a buffer from the slab header delayed list and put * it on the slab free list. */ static void wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer *buf) { struct _WsbmSlab *slab = buf->parent; struct _WsbmSlabSizeHeader *header = slab->header; struct _WsbmListHead *list = &buf->head; WSBMLISTDEL(list); WSBMLISTADDTAIL(list, &slab->freeBuffers); slab->numFree++; if (slab->head.next == &slab->head) WSBMLISTADDTAIL(&slab->head, &header->slabs); if (slab->numFree == slab->numBuffers) { list = &slab->head; WSBMLISTDEL(list); WSBMLISTADDTAIL(list, &header->freeSlabs); } if (header->slabs.next == &header->slabs || slab->numFree != slab->numBuffers) { struct _WsbmListHead *next; struct _WsbmSlabCache *cache = header->slabPool->cache; WSBMLISTFOREACHSAFE(list, next, &header->freeSlabs) { uint32_t i; struct _WsbmSlabBuffer *sBuf; slab = WSBMLISTENTRY(list, struct _WsbmSlab, head); WSBMLISTDELINIT(list); sBuf = slab->buffers; for (i = 0; i < slab->numBuffers; ++i) { if (sBuf->parent == slab) { WSBM_COND_FREE(&sBuf->event); wsbmBufStorageTakedown(&sBuf->storage); } sBuf++; } wsbmSetKernelBOFree(cache, slab->kbo); free(slab->buffers); free(slab); } } } static void wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader *header, int wait) { struct _WsbmListHead *list, *prev, *first, *head; struct _WsbmSlabBuffer *sBuf; struct _WsbmSlab *slab; int firstWasSignaled = 1; int signaled; uint32_t i; int ret; /* * Rerun the freeing test if the youngest tested buffer * was signaled, since there might be more idle buffers * in the delay list. */ while (firstWasSignaled) { firstWasSignaled = 0; signaled = 0; first = header->delayedBuffers.next; /* Only examine the oldest 1/3 of delayed buffers: */ if (header->numDelayed > 3) { for (i = 0; i < header->numDelayed; i += 3) { first = first->next; } } /* * No need to take the buffer mutex for each buffer we loop * through since we're currently the only user. */ head = first->next; WSBMLISTFOREACHPREVSAFE(list, prev, head) { if (list == &header->delayedBuffers) break; sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); slab = sBuf->parent; if (!signaled) { if (wait) { ret = wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0); if (ret) break; signaled = 1; wait = 0; } else { signaled = wsbmFenceSignaled(sBuf->fence, sBuf->fenceType); #ifdef DEBUG_FENCESIGNALED fencesignaled++; #endif } if (signaled) { if (list == first) firstWasSignaled = 1; wsbmFenceUnreference(&sBuf->fence); header->numDelayed--; wsbmSlabFreeBufferLocked(sBuf); } else break; } else if (wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) { wsbmFenceUnreference(&sBuf->fence); header->numDelayed--; wsbmSlabFreeBufferLocked(sBuf); } } } } static struct _WsbmSlabBuffer * wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header) { static struct _WsbmSlabBuffer *buf; struct _WsbmSlab *slab; struct _WsbmListHead *list; int count = WSBM_SLABPOOL_ALLOC_RETRIES; WSBM_MUTEX_LOCK(&header->mutex); while (header->slabs.next == &header->slabs && count > 0) { wsbmSlabCheckFreeLocked(header, 0); if (header->slabs.next != &header->slabs) break; WSBM_MUTEX_UNLOCK(&header->mutex); if (count != WSBM_SLABPOOL_ALLOC_RETRIES) usleep(1000); WSBM_MUTEX_LOCK(&header->mutex); (void)wsbmAllocSlab(header); count--; } list = header->slabs.next; if (list == &header->slabs) { WSBM_MUTEX_UNLOCK(&header->mutex); return NULL; } slab = WSBMLISTENTRY(list, struct _WsbmSlab, head); if (--slab->numFree == 0) WSBMLISTDELINIT(list); list = slab->freeBuffers.next; WSBMLISTDELINIT(list); WSBM_MUTEX_UNLOCK(&header->mutex); buf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); buf->storage.destroyContainer = NULL; #ifdef DEBUG_FENCESIGNALED createbuffer++; #endif return buf; } static struct _WsbmBufStorage * pool_create(struct _WsbmBufferPool *pool, unsigned long size, uint32_t placement, unsigned alignment) { struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool); struct _WsbmSlabSizeHeader *header; struct _WsbmSlabBuffer *sBuf; uint32_t i; int ret; /* * FIXME: Check for compatibility. */ header = slabPool->headers; for (i = 0; i < slabPool->numBuckets; ++i) { if (header->bufSize >= size) break; header++; } if (i < slabPool->numBuckets) { sBuf = wsbmSlabAllocBuffer(header); return ((sBuf) ? &sBuf->storage : NULL); } /* * Fall back to allocate a buffer object directly from DRM. * and wrap it in a wsbmBO structure. */ sBuf = calloc(1, sizeof(*sBuf)); if (!sBuf) return NULL; if (alignment) { if ((alignment < slabPool->pageSize) && (slabPool->pageSize % alignment)) goto out_err0; if ((alignment > slabPool->pageSize) && (alignment % slabPool->pageSize)) goto out_err0; } ret = wsbmBufStorageInit(&sBuf->storage, pool); if (ret) goto out_err0; ret = WSBM_COND_INIT(&sBuf->event); if (ret) goto out_err1; { union ttm_pl_create_arg arg; arg.req.size = size; arg.req.placement = placement; arg.req.page_alignment = alignment / slabPool->pageSize; DRMRESTARTCOMMANDWRITEREAD(pool->fd, slabPool->devOffset + TTM_PL_CREATE, arg, ret); if (ret) goto out_err2; sBuf->kBuf.gpuOffset = arg.rep.gpu_offset; sBuf->kBuf.placement = arg.rep.placement; sBuf->kBuf.handle = arg.rep.handle; sBuf->mapHandle = arg.rep.map_handle; sBuf->requestedSize = size; sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, pool->fd, sBuf->mapHandle); if (sBuf->virtual == MAP_FAILED) goto out_err3; } wsbmAtomicSet(&sBuf->writers, 0); return &sBuf->storage; out_err3: { struct ttm_pl_reference_req arg; arg.handle = sBuf->kBuf.handle; (void)drmCommandWriteRead(pool->fd, slabPool->devOffset + TTM_PL_UNREF, &arg, sizeof(arg)); } out_err2: WSBM_COND_FREE(&sBuf->event); out_err1: wsbmBufStorageTakedown(&sBuf->storage); out_err0: free(sBuf); return NULL; } static void pool_destroy(struct _WsbmBufStorage **p_buf) { struct _WsbmBufStorage *buf = *p_buf; struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); struct _WsbmSlab *slab; struct _WsbmSlabSizeHeader *header; *p_buf = NULL; if (!sBuf->isSlabBuffer) { struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf); struct ttm_pl_reference_req arg; if (sBuf->virtual != NULL) { (void)munmap(sBuf->virtual, sBuf->requestedSize); sBuf->virtual = NULL; } arg.handle = sBuf->kBuf.handle; (void)drmCommandWrite(slabPool->pool.fd, slabPool->devOffset + TTM_PL_UNREF, &arg, sizeof(arg)); WSBM_COND_FREE(&sBuf->event); wsbmBufStorageTakedown(&sBuf->storage); free(sBuf); return; } slab = sBuf->parent; header = slab->header; /* * No need to take the buffer mutex below since we're the only user. */ WSBM_MUTEX_LOCK(&header->mutex); sBuf->unFenced = 0; wsbmAtomicSet(&sBuf->writers, 0); wsbmAtomicSet(&sBuf->storage.refCount, 1); if (sBuf->fence && !wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) { WSBMLISTADDTAIL(&sBuf->head, &header->delayedBuffers); header->numDelayed++; } else { if (sBuf->fence) wsbmFenceUnreference(&sBuf->fence); wsbmSlabFreeBufferLocked(sBuf); } WSBM_MUTEX_UNLOCK(&header->mutex); } static void waitIdleLocked(struct _WsbmSlabBuffer *sBuf, int lazy) { struct _WsbmBufStorage *storage = &sBuf->storage; while (sBuf->unFenced || sBuf->fence != NULL) { if (sBuf->unFenced) WSBM_COND_WAIT(&sBuf->event, &storage->mutex); if (sBuf->fence != NULL) { if (!wsbmFenceSignaled(sBuf->fence, sBuf->fenceType)) { struct _WsbmFenceObject *fence = wsbmFenceReference(sBuf->fence); WSBM_MUTEX_UNLOCK(&storage->mutex); (void)wsbmFenceFinish(fence, sBuf->fenceType, lazy); WSBM_MUTEX_LOCK(&storage->mutex); if (sBuf->fence == fence) wsbmFenceUnreference(&sBuf->fence); wsbmFenceUnreference(&fence); } else { wsbmFenceUnreference(&sBuf->fence); } } } } static int pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) { struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); WSBM_MUTEX_LOCK(&buf->mutex); waitIdleLocked(sBuf, lazy); WSBM_MUTEX_UNLOCK(&buf->mutex); return 0; } static int pool_map(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)), void **virtual) { struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); *virtual = sBuf->virtual; return 0; } static void pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused))) { struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); if (wsbmAtomicDecZero(&sBuf->writers)) WSBM_COND_BROADCAST(&sBuf->event); } static int pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode) { struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); int ret = 0; WSBM_MUTEX_LOCK(&buf->mutex); if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) { int signaled; if (sBuf->unFenced) { ret = -EBUSY; goto out_unlock; } if (sBuf->isSlabBuffer) signaled = (sBuf->fence == NULL) || wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType); else signaled = (sBuf->fence == NULL) || wsbmFenceSignaled(sBuf->fence, sBuf->fenceType); ret = 0; if (signaled) { wsbmFenceUnreference(&sBuf->fence); wsbmAtomicInc(&sBuf->writers); } else ret = -EBUSY; goto out_unlock; } waitIdleLocked(sBuf, 0); wsbmAtomicInc(&sBuf->writers); out_unlock: WSBM_MUTEX_UNLOCK(&buf->mutex); return ret; } static void pool_unmap(struct _WsbmBufStorage *buf __attribute__ ((unused))) { ; } static unsigned long pool_poolOffset(struct _WsbmBufStorage *buf) { struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); return sBuf->start; } static unsigned long pool_size(struct _WsbmBufStorage *buf) { struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); if (!sBuf->isSlabBuffer) return sBuf->requestedSize; return sBuf->parent->header->bufSize; } static struct _WsbmKernelBuf * pool_kernel(struct _WsbmBufStorage *buf) { struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf; } static unsigned long pool_offset(struct _WsbmBufStorage *buf) { return pool_kernel(buf)->gpuOffset + pool_poolOffset(buf); } static void pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) { struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); struct _WsbmKernelBuf *kBuf; WSBM_MUTEX_LOCK(&buf->mutex); if (sBuf->fence) wsbmFenceUnreference(&sBuf->fence); kBuf = pool_kernel(buf); sBuf->fenceType = kBuf->fence_type_mask; if (!wsbmFenceSignaledCached(fence, sBuf->fenceType)) sBuf->fence = wsbmFenceReference(fence); sBuf->unFenced = 0; WSBM_COND_BROADCAST(&sBuf->event); WSBM_MUTEX_UNLOCK(&buf->mutex); } static int pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags __attribute__ ((unused)), uint64_t clr_flags __attribute__ ((unused))) { struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); WSBM_MUTEX_LOCK(&buf->mutex); while (wsbmAtomicRead(&sBuf->writers) != 0) { WSBM_COND_WAIT(&sBuf->event, &buf->mutex); } sBuf->unFenced = 1; WSBM_MUTEX_UNLOCK(&buf->mutex); return 0; } static void pool_unvalidate(struct _WsbmBufStorage *buf) { struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); WSBM_MUTEX_LOCK(&buf->mutex); if (sBuf->unFenced) { sBuf->unFenced = 0; WSBM_COND_BROADCAST(&sBuf->event); } WSBM_MUTEX_UNLOCK(&buf->mutex); } struct _WsbmSlabCache * wsbmSlabCacheInit(uint32_t checkIntervalMsec, uint32_t slabTimeoutMsec) { struct _WsbmSlabCache *tmp; tmp = calloc(1, sizeof(*tmp)); if (!tmp) return NULL; WSBM_MUTEX_INIT(&tmp->mutex); WSBM_MUTEX_LOCK(&tmp->mutex); tmp->slabTimeout.tv_usec = slabTimeoutMsec * 1000; tmp->slabTimeout.tv_sec = tmp->slabTimeout.tv_usec / 1000000; tmp->slabTimeout.tv_usec -= tmp->slabTimeout.tv_sec * 1000000; tmp->checkInterval.tv_usec = checkIntervalMsec * 1000; tmp->checkInterval.tv_sec = tmp->checkInterval.tv_usec / 1000000; tmp->checkInterval.tv_usec -= tmp->checkInterval.tv_sec * 1000000; gettimeofday(&tmp->nextCheck, NULL); wsbmTimeAdd(&tmp->nextCheck, &tmp->checkInterval); WSBMINITLISTHEAD(&tmp->timeoutList); WSBMINITLISTHEAD(&tmp->unCached); WSBMINITLISTHEAD(&tmp->cached); WSBM_MUTEX_UNLOCK(&tmp->mutex); return tmp; } void wsbmSlabCacheFinish(struct _WsbmSlabCache *cache) { struct timeval time; time = cache->nextCheck; WSBM_MUTEX_LOCK(&cache->mutex); wsbmTimeAdd(&time, &cache->checkInterval); wsbmFreeTimeoutKBOsLocked(cache, &time); WSBM_MUTEX_UNLOCK(&cache->mutex); assert(cache->timeoutList.next == &cache->timeoutList); assert(cache->unCached.next == &cache->unCached); assert(cache->cached.next == &cache->cached); WSBM_MUTEX_FREE(&cache->mutex); free(cache); } static void wsbmInitSizeHeader(struct _WsbmSlabPool *slabPool, uint32_t size, struct _WsbmSlabSizeHeader *header) { WSBM_MUTEX_INIT(&header->mutex); WSBM_MUTEX_LOCK(&header->mutex); WSBMINITLISTHEAD(&header->slabs); WSBMINITLISTHEAD(&header->freeSlabs); WSBMINITLISTHEAD(&header->delayedBuffers); header->numDelayed = 0; header->slabPool = slabPool; header->bufSize = size; WSBM_MUTEX_UNLOCK(&header->mutex); } static void wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader *header) { struct _WsbmListHead *list, *next; struct _WsbmSlabBuffer *sBuf; WSBM_MUTEX_LOCK(&header->mutex); WSBMLISTFOREACHSAFE(list, next, &header->delayedBuffers) { sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); if (sBuf->fence) { (void)wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0); wsbmFenceUnreference(&sBuf->fence); } header->numDelayed--; wsbmSlabFreeBufferLocked(sBuf); } WSBM_MUTEX_UNLOCK(&header->mutex); WSBM_MUTEX_FREE(&header->mutex); } static void pool_takedown(struct _WsbmBufferPool *pool) { struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool); unsigned int i; for (i = 0; i < slabPool->numBuckets; ++i) { wsbmFinishSizeHeader(&slabPool->headers[i]); } free(slabPool->headers); free(slabPool->bucketSizes); free(slabPool); } struct _WsbmBufferPool * wsbmSlabPoolInit(int fd, uint32_t devOffset, uint32_t placement, uint32_t validMask, uint32_t smallestSize, uint32_t numSizes, uint32_t desiredNumBuffers, uint32_t maxSlabSize, uint32_t pageAlignment, struct _WsbmSlabCache *cache) { struct _WsbmBufferPool *pool; struct _WsbmSlabPool *slabPool; uint32_t i; slabPool = calloc(1, sizeof(*slabPool)); if (!slabPool) return NULL; pool = &slabPool->pool; slabPool->bucketSizes = calloc(numSizes, sizeof(*slabPool->bucketSizes)); if (!slabPool->bucketSizes) goto out_err0; slabPool->headers = calloc(numSizes, sizeof(*slabPool->headers)); if (!slabPool->headers) goto out_err1; slabPool->devOffset = devOffset; slabPool->cache = cache; slabPool->proposedPlacement = placement; slabPool->validMask = validMask; slabPool->numBuckets = numSizes; slabPool->pageSize = getpagesize(); slabPool->pageAlignment = pageAlignment; slabPool->maxSlabSize = maxSlabSize; slabPool->desiredNumBuffers = desiredNumBuffers; for (i = 0; i < slabPool->numBuckets; ++i) { slabPool->bucketSizes[i] = (smallestSize << i); wsbmInitSizeHeader(slabPool, slabPool->bucketSizes[i], &slabPool->headers[i]); } pool->fd = fd; pool->map = &pool_map; pool->unmap = &pool_unmap; pool->destroy = &pool_destroy; pool->offset = &pool_offset; pool->poolOffset = &pool_poolOffset; pool->size = &pool_size; pool->create = &pool_create; pool->fence = &pool_fence; pool->kernel = &pool_kernel; pool->validate = &pool_validate; pool->unvalidate = &pool_unvalidate; pool->waitIdle = &pool_waitIdle; pool->takeDown = &pool_takedown; pool->releasefromcpu = &pool_releaseFromCpu; pool->syncforcpu = &pool_syncForCpu; return pool; out_err1: free(slabPool->bucketSizes); out_err0: free(slabPool); return NULL; }