xref: /hardware/intel/common/libwsbm/src/wsbm_slabpool.c

/**************************************************************************
 *
 * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA
 * All Rights Reserved.
 * Copyright 2009 VMware, Inc., Palo Alto, CA., USA
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 **************************************************************************/
/*
 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <drm/psb_ttm_placement_user.h>
#include <stdint.h>
#include <sys/time.h>
#include <errno.h>
#include <unistd.h>
#include <assert.h>
#include <string.h>
#include <sys/mman.h>
#include <xf86drm.h>
#include "wsbm_pool.h"
#include "wsbm_fencemgr.h"
#include "wsbm_priv.h"
#include "wsbm_manager.h"

#define WSBM_SLABPOOL_ALLOC_RETRIES 100
#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret)			\
	do {								\
		(_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \
	} while ((_ret) == -EAGAIN || (_ret) == -ERESTART);		\

#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret)		\
	do {								\
		(_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \
	} while ((_ret) == -EAGAIN || (_ret) == -ERESTART);		\


#ifdef DEBUG_FENCESIGNALED
static int createbuffer = 0;
static int fencesignaled = 0;
#endif

struct _WsbmSlab;

struct _WsbmSlabBuffer
{
    struct _WsbmKernelBuf kBuf;
    struct _WsbmBufStorage storage;
    struct _WsbmCond event;

    /*
     * Remains constant after creation.
     */

    int isSlabBuffer;
    struct _WsbmSlab *parent;
    uint32_t start;
    void *virtual;
    unsigned long requestedSize;
    uint64_t mapHandle;

    /*
     * Protected by struct _WsbmSlabSizeHeader::mutex
     */

    struct _WsbmListHead head;

    /*
     * Protected by this::mutex
     */

    struct _WsbmFenceObject *fence;
    uint32_t fenceType;
    struct _WsbmAtomic writers;	       /* (Only upping) */
    int unFenced;
};

struct _WsbmSlabPool;
struct _WsbmSlabKernelBO
{

    /*
     * Constant at creation
     */

    struct _WsbmKernelBuf kBuf;
    uint32_t pageAlignment;
    void *virtual;
    unsigned long actualSize;
    uint64_t mapHandle;

    /*
     * Protected by struct _WsbmSlabCache::mutex
     */

    struct _WsbmSlabPool *slabPool;
    uint32_t proposedPlacement;
    struct _WsbmListHead timeoutHead;
    struct _WsbmListHead head;
    struct timeval timeFreed;
};

struct _WsbmSlab
{
    struct _WsbmListHead head;
    struct _WsbmListHead freeBuffers;
    uint32_t numBuffers;
    uint32_t numFree;
    struct _WsbmSlabBuffer *buffers;
    struct _WsbmSlabSizeHeader *header;
    struct _WsbmSlabKernelBO *kbo;
};

struct _WsbmSlabSizeHeader
{
    /*
     * Constant at creation.
     */
    struct _WsbmSlabPool *slabPool;
    uint32_t bufSize;

    /*
     * Protected by this::mutex
     */

    struct _WsbmListHead slabs;
    struct _WsbmListHead freeSlabs;
    struct _WsbmListHead delayedBuffers;
    uint32_t numDelayed;
    struct _WsbmMutex mutex;
};

struct _WsbmSlabCache
{
    struct timeval slabTimeout;
    struct timeval checkInterval;
    struct timeval nextCheck;
    struct _WsbmListHead timeoutList;
    struct _WsbmListHead unCached;
    struct _WsbmListHead cached;
    struct _WsbmMutex mutex;
};

struct _WsbmSlabPool
{
    struct _WsbmBufferPool pool;

    /*
     * The data of this structure remains constant after
     * initialization and thus needs no mutex protection.
     */

    unsigned int devOffset;
    struct _WsbmSlabCache *cache;
    uint32_t proposedPlacement;
    uint32_t validMask;
    uint32_t *bucketSizes;
    uint32_t numBuckets;
    uint32_t pageSize;
    int pageAlignment;
    int maxSlabSize;
    int desiredNumBuffers;
    struct _WsbmSlabSizeHeader *headers;
};

static inline struct _WsbmSlabPool *
slabPoolFromPool(struct _WsbmBufferPool *pool)
{
    return containerOf(pool, struct _WsbmSlabPool, pool);
}

static inline struct _WsbmSlabPool *
slabPoolFromBuf(struct _WsbmSlabBuffer *sBuf)
{
    return slabPoolFromPool(sBuf->storage.pool);
}

static inline struct _WsbmSlabBuffer *
slabBuffer(struct _WsbmBufStorage *buf)
{
    return containerOf(buf, struct _WsbmSlabBuffer, storage);
}

/*
 * FIXME: Perhaps arrange timeout slabs in size buckets for fast
 * retreival??
 */

static inline int
wsbmTimeAfterEq(struct timeval *arg1, struct timeval *arg2)
{
    return ((arg1->tv_sec > arg2->tv_sec) ||
	    ((arg1->tv_sec == arg2->tv_sec) &&
	     (arg1->tv_usec > arg2->tv_usec)));
}

static inline void
wsbmTimeAdd(struct timeval *arg, struct timeval *add)
{
    unsigned int sec;

    arg->tv_sec += add->tv_sec;
    arg->tv_usec += add->tv_usec;
    sec = arg->tv_usec / 1000000;
    arg->tv_sec += sec;
    arg->tv_usec -= sec * 1000000;
}

static void
wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo)
{
    struct ttm_pl_reference_req arg;
    struct _WsbmSlabPool *slabPool;

    if (!kbo)
	return;

    slabPool = kbo->slabPool;
    arg.handle = kbo->kBuf.handle;
    (void)munmap(kbo->virtual, kbo->actualSize);
    (void)drmCommandWrite(slabPool->pool.fd,
			  slabPool->devOffset + TTM_PL_UNREF, &arg,
			  sizeof(arg));
    free(kbo);
}

static void
wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache *cache, struct timeval *time)
{
    struct _WsbmListHead *list, *next;
    struct _WsbmSlabKernelBO *kbo;

    if (!wsbmTimeAfterEq(time, &cache->nextCheck))
	return;

    WSBMLISTFOREACHSAFE(list, next, &cache->timeoutList) {
	kbo = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, timeoutHead);

	if (!wsbmTimeAfterEq(time, &kbo->timeFreed))
	    break;

	WSBMLISTDELINIT(&kbo->timeoutHead);
	WSBMLISTDELINIT(&kbo->head);
	wsbmFreeKernelBO(kbo);
    }

    cache->nextCheck = *time;
    wsbmTimeAdd(&cache->nextCheck, &cache->checkInterval);
}

/*
 * Add a _SlabKernelBO to the free slab manager.
 * This means that it is available for reuse, but if it's not
 * reused in a while, it will be freed.
 */

static void
wsbmSetKernelBOFree(struct _WsbmSlabCache *cache,
		    struct _WsbmSlabKernelBO *kbo)
{
    struct timeval time;
    struct timeval timeFreed;

    gettimeofday(&time, NULL);
    timeFreed = time;
    WSBM_MUTEX_LOCK(&cache->mutex);
    wsbmTimeAdd(&timeFreed, &cache->slabTimeout);
    kbo->timeFreed = timeFreed;

    if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED)
	WSBMLISTADD(&kbo->head, &cache->cached);
    else
	WSBMLISTADD(&kbo->head, &cache->unCached);

    WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList);
    wsbmFreeTimeoutKBOsLocked(cache, &time);

    WSBM_MUTEX_UNLOCK(&cache->mutex);
}

/*
 * Get a _SlabKernelBO for us to use as storage for a slab.
 */

static struct _WsbmSlabKernelBO *
wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header)
{
    struct _WsbmSlabPool *slabPool = header->slabPool;
    struct _WsbmSlabCache *cache = slabPool->cache;
    struct _WsbmListHead *list, *head;
    uint32_t size = header->bufSize * slabPool->desiredNumBuffers;
    struct _WsbmSlabKernelBO *kbo;
    struct _WsbmSlabKernelBO *kboTmp;
    int ret;

    /*
     * FIXME: We should perhaps allow some variation in slabsize in order
     * to efficiently reuse slabs.
     */

    size = (size <= (uint32_t) slabPool->maxSlabSize) ? size : (uint32_t) slabPool->maxSlabSize;
    if (size < header->bufSize)
	size = header->bufSize;
    size = (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1);
    WSBM_MUTEX_LOCK(&cache->mutex);

    kbo = NULL;

  retry:
    head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ?
	&cache->cached : &cache->unCached;

    WSBMLISTFOREACH(list, head) {
	kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head);

	if ((kboTmp->actualSize == size) &&
	    (slabPool->pageAlignment == 0 ||
	     (kboTmp->pageAlignment % slabPool->pageAlignment) == 0)) {

	    if (!kbo)
		kbo = kboTmp;

	    if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0)
		break;

	}
    }

    if (kbo) {
	WSBMLISTDELINIT(&kbo->head);
	WSBMLISTDELINIT(&kbo->timeoutHead);
    }

    WSBM_MUTEX_UNLOCK(&cache->mutex);

    if (kbo) {
	uint32_t new_mask =
	    kbo->proposedPlacement ^ slabPool->proposedPlacement;

	ret = 0;
	if (new_mask) {
	    union ttm_pl_setstatus_arg arg;
	    struct ttm_pl_setstatus_req *req = &arg.req;
	    struct ttm_pl_rep *rep = &arg.rep;

	    req->handle = kbo->kBuf.handle;
	    req->set_placement = slabPool->proposedPlacement & new_mask;
	    req->clr_placement = ~slabPool->proposedPlacement & new_mask;
	    DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
				       slabPool->devOffset + TTM_PL_SETSTATUS,
				       arg, ret);
	    if (ret == 0) {
		kbo->kBuf.gpuOffset = rep->gpu_offset;
		kbo->kBuf.placement = rep->placement;
	    }
	    kbo->proposedPlacement = slabPool->proposedPlacement;
	}

	if (ret == 0)
	    return kbo;

	wsbmFreeKernelBO(kbo);
	kbo = NULL;
	goto retry;
    }

    kbo = calloc(1, sizeof(*kbo));
    if (!kbo)
	return NULL;

    {
	union ttm_pl_create_arg arg;

	kbo->slabPool = slabPool;
	WSBMINITLISTHEAD(&kbo->head);
	WSBMINITLISTHEAD(&kbo->timeoutHead);

	arg.req.size = size;
	arg.req.placement = slabPool->proposedPlacement;
	arg.req.page_alignment = slabPool->pageAlignment;

	DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
				   slabPool->devOffset + TTM_PL_CREATE,
				   arg, ret);
	if (ret)
	    goto out_err0;

	kbo->kBuf.gpuOffset = arg.rep.gpu_offset;
	kbo->kBuf.placement = arg.rep.placement;
	kbo->kBuf.handle = arg.rep.handle;

	kbo->actualSize = arg.rep.bo_size;
	kbo->mapHandle = arg.rep.map_handle;
	kbo->proposedPlacement = slabPool->proposedPlacement;
    }

    kbo->virtual = mmap(0, kbo->actualSize,
			PROT_READ | PROT_WRITE, MAP_SHARED,
			slabPool->pool.fd, kbo->mapHandle);

    if (kbo->virtual == MAP_FAILED) {
	ret = -errno;
	goto out_err1;
    }

    return kbo;

  out_err1:
    {
	struct ttm_pl_reference_req arg = {.handle = kbo->kBuf.handle };

	(void)drmCommandWrite(slabPool->pool.fd,
			      slabPool->devOffset + TTM_PL_UNREF,
			      &arg, sizeof(arg));
    }
  out_err0:
    free(kbo);
    return NULL;
}

static int
wsbmAllocSlab(struct _WsbmSlabSizeHeader *header)
{
    struct _WsbmSlab *slab;
    struct _WsbmSlabBuffer *sBuf;
    uint32_t numBuffers;
    uint32_t ret;
    uint32_t i;

    slab = calloc(1, sizeof(*slab));
    if (!slab)
	return -ENOMEM;

    slab->kbo = wsbmAllocKernelBO(header);
    if (!slab->kbo) {
	ret = -ENOMEM;
	goto out_err0;
    }

    numBuffers = slab->kbo->actualSize / header->bufSize;
    if (!numBuffers) {
        ret = -ENOMEM;
        goto out_err1;
    }

    slab->buffers = calloc(numBuffers, sizeof(*slab->buffers));
    if (!slab->buffers) {
	ret = -ENOMEM;
	goto out_err1;
    }

    WSBMINITLISTHEAD(&slab->head);
    WSBMINITLISTHEAD(&slab->freeBuffers);
    slab->numBuffers = numBuffers;
    slab->numFree = 0;
    slab->header = header;

    sBuf = slab->buffers;
    for (i = 0; i < numBuffers; ++i) {
	ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool);
	if (ret)
	    goto out_err2;
	sBuf->parent = slab;
	sBuf->start = i * header->bufSize;
	sBuf->virtual = (void *)((uint8_t *) slab->kbo->virtual +
				 sBuf->start);
	wsbmAtomicSet(&sBuf->writers, 0);
	sBuf->isSlabBuffer = 1;
	WSBM_COND_INIT(&sBuf->event);
	WSBMLISTADDTAIL(&sBuf->head, &slab->freeBuffers);
	slab->numFree++;
	sBuf++;
    }

    WSBMLISTADDTAIL(&slab->head, &header->slabs);

    return 0;

  out_err2:
    sBuf = slab->buffers;
    for (i = 0; i < numBuffers; ++i) {
	if (sBuf->parent == slab) {
	    WSBM_COND_FREE(&sBuf->event);
	    wsbmBufStorageTakedown(&sBuf->storage);
	}
	sBuf++;
    }
    free(slab->buffers);
  out_err1:
    wsbmSetKernelBOFree(header->slabPool->cache, slab->kbo);
  out_err0:
    free(slab);
    return ret;
}

/*
 * Delete a buffer from the slab header delayed list and put
 * it on the slab free list.
 */

static void
wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer *buf)
{
    struct _WsbmSlab *slab = buf->parent;
    struct _WsbmSlabSizeHeader *header = slab->header;
    struct _WsbmListHead *list = &buf->head;

    WSBMLISTDEL(list);
    WSBMLISTADDTAIL(list, &slab->freeBuffers);
    slab->numFree++;

    if (slab->head.next == &slab->head)
	WSBMLISTADDTAIL(&slab->head, &header->slabs);

    if (slab->numFree == slab->numBuffers) {
	list = &slab->head;
	WSBMLISTDEL(list);
	WSBMLISTADDTAIL(list, &header->freeSlabs);
    }

    if (header->slabs.next == &header->slabs ||
	slab->numFree != slab->numBuffers) {

	struct _WsbmListHead *next;
	struct _WsbmSlabCache *cache = header->slabPool->cache;

	WSBMLISTFOREACHSAFE(list, next, &header->freeSlabs) {
	    uint32_t i;
	    struct _WsbmSlabBuffer *sBuf;

	    slab = WSBMLISTENTRY(list, struct _WsbmSlab, head);

	    WSBMLISTDELINIT(list);

	    sBuf = slab->buffers;
	    for (i = 0; i < slab->numBuffers; ++i) {
		if (sBuf->parent == slab) {
		    WSBM_COND_FREE(&sBuf->event);
		    wsbmBufStorageTakedown(&sBuf->storage);
		}
		sBuf++;
	    }
	    wsbmSetKernelBOFree(cache, slab->kbo);
	    free(slab->buffers);
	    free(slab);
	}
    }
}

static void
wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader *header, int wait)
{
  struct _WsbmListHead *list, *prev, *first, *head;
    struct _WsbmSlabBuffer *sBuf;
    struct _WsbmSlab *slab;
    int firstWasSignaled = 1;
    int signaled;
    uint32_t i;
    int ret;

    /*
     * Rerun the freeing test if the youngest tested buffer
     * was signaled, since there might be more idle buffers
     * in the delay list.
     */

    while (firstWasSignaled) {
	firstWasSignaled = 0;
	signaled = 0;
	first = header->delayedBuffers.next;

	/* Only examine the oldest 1/3 of delayed buffers:
	 */
	if (header->numDelayed > 3) {
	    for (i = 0; i < header->numDelayed; i += 3) {
		first = first->next;
	    }
	}

	/*
	 * No need to take the buffer mutex for each buffer we loop
	 * through since we're currently the only user.
	 */

	head = first->next;
	WSBMLISTFOREACHPREVSAFE(list, prev, head) {

	    if (list == &header->delayedBuffers)
		break;

	    sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);

	    slab = sBuf->parent;

	    if (!signaled) {
		if (wait) {
		    ret = wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);
		    if (ret)
			break;
		    signaled = 1;
		    wait = 0;
		} else {
		    signaled =
			wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);
#ifdef DEBUG_FENCESIGNALED
		    fencesignaled++;
#endif
		}
		if (signaled) {
		    if (list == first)
			firstWasSignaled = 1;
		    wsbmFenceUnreference(&sBuf->fence);
		    header->numDelayed--;
		    wsbmSlabFreeBufferLocked(sBuf);
		} else
		    break;
	    } else if (wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {
		wsbmFenceUnreference(&sBuf->fence);
		header->numDelayed--;
		wsbmSlabFreeBufferLocked(sBuf);
	    }
	}
    }
}

static struct _WsbmSlabBuffer *
wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header)
{
    static struct _WsbmSlabBuffer *buf;
    struct _WsbmSlab *slab;
    struct _WsbmListHead *list;
    int count = WSBM_SLABPOOL_ALLOC_RETRIES;

    WSBM_MUTEX_LOCK(&header->mutex);
    while (header->slabs.next == &header->slabs && count > 0) {
	wsbmSlabCheckFreeLocked(header, 0);
	if (header->slabs.next != &header->slabs)
	    break;

	WSBM_MUTEX_UNLOCK(&header->mutex);
	if (count != WSBM_SLABPOOL_ALLOC_RETRIES)
	    usleep(1000);
	WSBM_MUTEX_LOCK(&header->mutex);
	(void)wsbmAllocSlab(header);
	count--;
    }

    list = header->slabs.next;
    if (list == &header->slabs) {
	WSBM_MUTEX_UNLOCK(&header->mutex);
	return NULL;
    }
    slab = WSBMLISTENTRY(list, struct _WsbmSlab, head);
    if (--slab->numFree == 0)
	WSBMLISTDELINIT(list);

    list = slab->freeBuffers.next;
    WSBMLISTDELINIT(list);

    WSBM_MUTEX_UNLOCK(&header->mutex);
    buf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);

    buf->storage.destroyContainer = NULL;

#ifdef DEBUG_FENCESIGNALED
    createbuffer++;
#endif
    return buf;
}

static struct _WsbmBufStorage *
pool_create(struct _WsbmBufferPool *pool, unsigned long size,
	    uint32_t placement, unsigned alignment)
{
    struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool);
    struct _WsbmSlabSizeHeader *header;
    struct _WsbmSlabBuffer *sBuf;
    uint32_t i;
    int ret;

    /*
     * FIXME: Check for compatibility.
     */

    header = slabPool->headers;
    for (i = 0; i < slabPool->numBuckets; ++i) {
	if (header->bufSize >= size)
	    break;
	header++;
    }

    if (i < slabPool->numBuckets) {
	sBuf = wsbmSlabAllocBuffer(header);
	return ((sBuf) ? &sBuf->storage : NULL);
    }

    /*
     * Fall back to allocate a buffer object directly from DRM.
     * and wrap it in a wsbmBO structure.
     */

    sBuf = calloc(1, sizeof(*sBuf));

    if (!sBuf)
	return NULL;

    if (alignment) {
	if ((alignment < slabPool->pageSize)
	    && (slabPool->pageSize % alignment))
	    goto out_err0;
	if ((alignment > slabPool->pageSize)
	    && (alignment % slabPool->pageSize))
	    goto out_err0;
    }

    ret = wsbmBufStorageInit(&sBuf->storage, pool);
    if (ret)
	goto out_err0;

    ret = WSBM_COND_INIT(&sBuf->event);
    if (ret)
	goto out_err1;

    {
	union ttm_pl_create_arg arg;

	arg.req.size = size;
	arg.req.placement = placement;
	arg.req.page_alignment = alignment / slabPool->pageSize;

	DRMRESTARTCOMMANDWRITEREAD(pool->fd,
				   slabPool->devOffset + TTM_PL_CREATE,
				   arg, ret);

	if (ret)
	    goto out_err2;

	sBuf->kBuf.gpuOffset = arg.rep.gpu_offset;
	sBuf->kBuf.placement = arg.rep.placement;
	sBuf->kBuf.handle = arg.rep.handle;
	sBuf->mapHandle = arg.rep.map_handle;
	sBuf->requestedSize = size;

	sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED,
			     pool->fd, sBuf->mapHandle);

	if (sBuf->virtual == MAP_FAILED)
	    goto out_err3;
    }

    wsbmAtomicSet(&sBuf->writers, 0);
    return &sBuf->storage;
  out_err3:
    {
	struct ttm_pl_reference_req arg;

	arg.handle = sBuf->kBuf.handle;
	(void)drmCommandWriteRead(pool->fd,
				  slabPool->devOffset + TTM_PL_UNREF,
				  &arg, sizeof(arg));
    }
  out_err2:
    WSBM_COND_FREE(&sBuf->event);
  out_err1:
    wsbmBufStorageTakedown(&sBuf->storage);
  out_err0:
    free(sBuf);
    return NULL;
}

static void
pool_destroy(struct _WsbmBufStorage **p_buf)
{
    struct _WsbmBufStorage *buf = *p_buf;
    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
    struct _WsbmSlab *slab;
    struct _WsbmSlabSizeHeader *header;

    *p_buf = NULL;

    if (!sBuf->isSlabBuffer) {
	struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf);
	struct ttm_pl_reference_req arg;

	if (sBuf->virtual != NULL) {
	    (void)munmap(sBuf->virtual, sBuf->requestedSize);
	    sBuf->virtual = NULL;
	}

	arg.handle = sBuf->kBuf.handle;
	(void)drmCommandWrite(slabPool->pool.fd,
			      slabPool->devOffset + TTM_PL_UNREF,
			      &arg, sizeof(arg));

	WSBM_COND_FREE(&sBuf->event);
	wsbmBufStorageTakedown(&sBuf->storage);
	free(sBuf);
	return;
    }

    slab = sBuf->parent;
    header = slab->header;

    /*
     * No need to take the buffer mutex below since we're the only user.
     */

    WSBM_MUTEX_LOCK(&header->mutex);
    sBuf->unFenced = 0;
    wsbmAtomicSet(&sBuf->writers, 0);
    wsbmAtomicSet(&sBuf->storage.refCount, 1);

    if (sBuf->fence && !wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {
	WSBMLISTADDTAIL(&sBuf->head, &header->delayedBuffers);
	header->numDelayed++;
    } else {
	if (sBuf->fence)
	    wsbmFenceUnreference(&sBuf->fence);
	wsbmSlabFreeBufferLocked(sBuf);
    }

    WSBM_MUTEX_UNLOCK(&header->mutex);
}

static void
waitIdleLocked(struct _WsbmSlabBuffer *sBuf, int lazy)
{
    struct _WsbmBufStorage *storage = &sBuf->storage;

    while (sBuf->unFenced || sBuf->fence != NULL) {

	if (sBuf->unFenced)
	    WSBM_COND_WAIT(&sBuf->event, &storage->mutex);

	if (sBuf->fence != NULL) {
	    if (!wsbmFenceSignaled(sBuf->fence, sBuf->fenceType)) {
		struct _WsbmFenceObject *fence =
		    wsbmFenceReference(sBuf->fence);

		WSBM_MUTEX_UNLOCK(&storage->mutex);
		(void)wsbmFenceFinish(fence, sBuf->fenceType, lazy);
		WSBM_MUTEX_LOCK(&storage->mutex);
		if (sBuf->fence == fence)
		    wsbmFenceUnreference(&sBuf->fence);

		wsbmFenceUnreference(&fence);
	    } else {
		wsbmFenceUnreference(&sBuf->fence);
	    }
	}
    }
}

static int
pool_waitIdle(struct _WsbmBufStorage *buf, int lazy)
{
    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);

    WSBM_MUTEX_LOCK(&buf->mutex);
    waitIdleLocked(sBuf, lazy);
    WSBM_MUTEX_UNLOCK(&buf->mutex);

    return 0;
}

static int
pool_map(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)), void **virtual)
{
    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);

    *virtual = sBuf->virtual;

    return 0;
}

static void
pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)))
{
    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);

    if (wsbmAtomicDecZero(&sBuf->writers))
	WSBM_COND_BROADCAST(&sBuf->event);
}

static int
pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode)
{
    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
    int ret = 0;

    WSBM_MUTEX_LOCK(&buf->mutex);
    if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) {
	int signaled;

	if (sBuf->unFenced) {
	    ret = -EBUSY;
	    goto out_unlock;
	}

	if (sBuf->isSlabBuffer)
	    signaled = (sBuf->fence == NULL) ||
		wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType);
	else
	    signaled = (sBuf->fence == NULL) ||
		wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);

	ret = 0;
	if (signaled) {
	    wsbmFenceUnreference(&sBuf->fence);
	    wsbmAtomicInc(&sBuf->writers);
	} else
	    ret = -EBUSY;
	goto out_unlock;
    }
    waitIdleLocked(sBuf, 0);
    wsbmAtomicInc(&sBuf->writers);
  out_unlock:
    WSBM_MUTEX_UNLOCK(&buf->mutex);
    return ret;
}

static void
pool_unmap(struct _WsbmBufStorage *buf __attribute__ ((unused)))
{
    ;
}

static unsigned long
pool_poolOffset(struct _WsbmBufStorage *buf)
{
    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);

    return sBuf->start;
}

static unsigned long
pool_size(struct _WsbmBufStorage *buf)
{
    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);

    if (!sBuf->isSlabBuffer)
	return sBuf->requestedSize;

    return sBuf->parent->header->bufSize;
}

static struct _WsbmKernelBuf *
pool_kernel(struct _WsbmBufStorage *buf)
{
    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);

    return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf;
}

static unsigned long
pool_offset(struct _WsbmBufStorage *buf)
{
    return pool_kernel(buf)->gpuOffset + pool_poolOffset(buf);
}

static void
pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence)
{
    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
    struct _WsbmKernelBuf *kBuf;

    WSBM_MUTEX_LOCK(&buf->mutex);
    if (sBuf->fence)
	wsbmFenceUnreference(&sBuf->fence);

    kBuf = pool_kernel(buf);
    sBuf->fenceType = kBuf->fence_type_mask;
    if (!wsbmFenceSignaledCached(fence, sBuf->fenceType))
	sBuf->fence = wsbmFenceReference(fence);

    sBuf->unFenced = 0;
    WSBM_COND_BROADCAST(&sBuf->event);
    WSBM_MUTEX_UNLOCK(&buf->mutex);
}

static int
pool_validate(struct _WsbmBufStorage *buf,
        uint64_t set_flags __attribute__ ((unused)), uint64_t clr_flags __attribute__ ((unused)))
{
    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);

    WSBM_MUTEX_LOCK(&buf->mutex);
    while (wsbmAtomicRead(&sBuf->writers) != 0) {
	WSBM_COND_WAIT(&sBuf->event, &buf->mutex);
    }

    sBuf->unFenced = 1;
    WSBM_MUTEX_UNLOCK(&buf->mutex);
    return 0;
}

static void
pool_unvalidate(struct _WsbmBufStorage *buf)
{
    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);

    WSBM_MUTEX_LOCK(&buf->mutex);
    if (sBuf->unFenced) {
	sBuf->unFenced = 0;
	WSBM_COND_BROADCAST(&sBuf->event);
    }
    WSBM_MUTEX_UNLOCK(&buf->mutex);
}

struct _WsbmSlabCache *
wsbmSlabCacheInit(uint32_t checkIntervalMsec, uint32_t slabTimeoutMsec)
{
    struct _WsbmSlabCache *tmp;

    tmp = calloc(1, sizeof(*tmp));
    if (!tmp)
	return NULL;

    WSBM_MUTEX_INIT(&tmp->mutex);
    WSBM_MUTEX_LOCK(&tmp->mutex);
    tmp->slabTimeout.tv_usec = slabTimeoutMsec * 1000;
    tmp->slabTimeout.tv_sec = tmp->slabTimeout.tv_usec / 1000000;
    tmp->slabTimeout.tv_usec -= tmp->slabTimeout.tv_sec * 1000000;

    tmp->checkInterval.tv_usec = checkIntervalMsec * 1000;
    tmp->checkInterval.tv_sec = tmp->checkInterval.tv_usec / 1000000;
    tmp->checkInterval.tv_usec -= tmp->checkInterval.tv_sec * 1000000;

    gettimeofday(&tmp->nextCheck, NULL);
    wsbmTimeAdd(&tmp->nextCheck, &tmp->checkInterval);
    WSBMINITLISTHEAD(&tmp->timeoutList);
    WSBMINITLISTHEAD(&tmp->unCached);
    WSBMINITLISTHEAD(&tmp->cached);
    WSBM_MUTEX_UNLOCK(&tmp->mutex);

    return tmp;
}

void
wsbmSlabCacheFinish(struct _WsbmSlabCache *cache)
{
    struct timeval time;

    time = cache->nextCheck;
    WSBM_MUTEX_LOCK(&cache->mutex);
    wsbmTimeAdd(&time, &cache->checkInterval);
    wsbmFreeTimeoutKBOsLocked(cache, &time);
    WSBM_MUTEX_UNLOCK(&cache->mutex);

    assert(cache->timeoutList.next == &cache->timeoutList);
    assert(cache->unCached.next == &cache->unCached);
    assert(cache->cached.next == &cache->cached);

    WSBM_MUTEX_FREE(&cache->mutex);
    free(cache);
}

static void
wsbmInitSizeHeader(struct _WsbmSlabPool *slabPool, uint32_t size,
		   struct _WsbmSlabSizeHeader *header)
{
    WSBM_MUTEX_INIT(&header->mutex);
    WSBM_MUTEX_LOCK(&header->mutex);

    WSBMINITLISTHEAD(&header->slabs);
    WSBMINITLISTHEAD(&header->freeSlabs);
    WSBMINITLISTHEAD(&header->delayedBuffers);

    header->numDelayed = 0;
    header->slabPool = slabPool;
    header->bufSize = size;

    WSBM_MUTEX_UNLOCK(&header->mutex);
}

static void
wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader *header)
{
    struct _WsbmListHead *list, *next;
    struct _WsbmSlabBuffer *sBuf;

    WSBM_MUTEX_LOCK(&header->mutex);
    WSBMLISTFOREACHSAFE(list, next, &header->delayedBuffers) {
	sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);

	if (sBuf->fence) {
	    (void)wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);
	    wsbmFenceUnreference(&sBuf->fence);
	}
	header->numDelayed--;
	wsbmSlabFreeBufferLocked(sBuf);
    }
    WSBM_MUTEX_UNLOCK(&header->mutex);
    WSBM_MUTEX_FREE(&header->mutex);
}

static void
pool_takedown(struct _WsbmBufferPool *pool)
{
    struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool);
    unsigned int i;

    for (i = 0; i < slabPool->numBuckets; ++i) {
	wsbmFinishSizeHeader(&slabPool->headers[i]);
    }

    free(slabPool->headers);
    free(slabPool->bucketSizes);
    free(slabPool);
}

struct _WsbmBufferPool *
wsbmSlabPoolInit(int fd,
		 uint32_t devOffset,
		 uint32_t placement,
		 uint32_t validMask,
		 uint32_t smallestSize,
		 uint32_t numSizes,
		 uint32_t desiredNumBuffers,
		 uint32_t maxSlabSize,
		 uint32_t pageAlignment, struct _WsbmSlabCache *cache)
{
    struct _WsbmBufferPool *pool;
    struct _WsbmSlabPool *slabPool;
    uint32_t i;

    slabPool = calloc(1, sizeof(*slabPool));
    if (!slabPool)
	return NULL;

    pool = &slabPool->pool;

    slabPool->bucketSizes = calloc(numSizes, sizeof(*slabPool->bucketSizes));
    if (!slabPool->bucketSizes)
	goto out_err0;

    slabPool->headers = calloc(numSizes, sizeof(*slabPool->headers));
    if (!slabPool->headers)
	goto out_err1;

    slabPool->devOffset = devOffset;
    slabPool->cache = cache;
    slabPool->proposedPlacement = placement;
    slabPool->validMask = validMask;
    slabPool->numBuckets = numSizes;
    slabPool->pageSize = getpagesize();
    slabPool->pageAlignment = pageAlignment;
    slabPool->maxSlabSize = maxSlabSize;
    slabPool->desiredNumBuffers = desiredNumBuffers;

    for (i = 0; i < slabPool->numBuckets; ++i) {
	slabPool->bucketSizes[i] = (smallestSize << i);
	wsbmInitSizeHeader(slabPool, slabPool->bucketSizes[i],
			   &slabPool->headers[i]);
    }

    pool->fd = fd;
    pool->map = &pool_map;
    pool->unmap = &pool_unmap;
    pool->destroy = &pool_destroy;
    pool->offset = &pool_offset;
    pool->poolOffset = &pool_poolOffset;
    pool->size = &pool_size;
    pool->create = &pool_create;
    pool->fence = &pool_fence;
    pool->kernel = &pool_kernel;
    pool->validate = &pool_validate;
    pool->unvalidate = &pool_unvalidate;
    pool->waitIdle = &pool_waitIdle;
    pool->takeDown = &pool_takedown;
    pool->releasefromcpu = &pool_releaseFromCpu;
    pool->syncforcpu = &pool_syncForCpu;

    return pool;

  out_err1:
    free(slabPool->bucketSizes);
  out_err0:
    free(slabPool);

    return NULL;
}