1/**************************************************************************
2 *
3 * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA
4 * All Rights Reserved.
5 * Copyright 2009 VMware, Inc., Palo Alto, CA., USA
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
23 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
24 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
25 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
26 * USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
28 **************************************************************************/
29/*
30 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
31 */
32
33#ifdef HAVE_CONFIG_H
34#include "config.h"
35#endif
36
37#include <drm/psb_ttm_placement_user.h>
38#include <stdint.h>
39#include <sys/time.h>
40#include <errno.h>
41#include <unistd.h>
42#include <assert.h>
43#include <string.h>
44#include <sys/mman.h>
45#include <xf86drm.h>
46#include "wsbm_pool.h"
47#include "wsbm_fencemgr.h"
48#include "wsbm_priv.h"
49#include "wsbm_manager.h"
50
51#define WSBM_SLABPOOL_ALLOC_RETRIES 100
52#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret)			\
53	do {								\
54		(_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \
55	} while ((_ret) == -EAGAIN || (_ret) == -ERESTART);		\
56
57#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret)		\
58	do {								\
59		(_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \
60	} while ((_ret) == -EAGAIN || (_ret) == -ERESTART);		\
61
62
63#ifdef DEBUG_FENCESIGNALED
64static int createbuffer = 0;
65static int fencesignaled = 0;
66#endif
67
68struct _WsbmSlab;
69
70struct _WsbmSlabBuffer
71{
72    struct _WsbmKernelBuf kBuf;
73    struct _WsbmBufStorage storage;
74    struct _WsbmCond event;
75
76    /*
77     * Remains constant after creation.
78     */
79
80    int isSlabBuffer;
81    struct _WsbmSlab *parent;
82    uint32_t start;
83    void *virtual;
84    unsigned long requestedSize;
85    uint64_t mapHandle;
86
87    /*
88     * Protected by struct _WsbmSlabSizeHeader::mutex
89     */
90
91    struct _WsbmListHead head;
92
93    /*
94     * Protected by this::mutex
95     */
96
97    struct _WsbmFenceObject *fence;
98    uint32_t fenceType;
99    struct _WsbmAtomic writers;	       /* (Only upping) */
100    int unFenced;
101};
102
103struct _WsbmSlabPool;
104struct _WsbmSlabKernelBO
105{
106
107    /*
108     * Constant at creation
109     */
110
111    struct _WsbmKernelBuf kBuf;
112    uint32_t pageAlignment;
113    void *virtual;
114    unsigned long actualSize;
115    uint64_t mapHandle;
116
117    /*
118     * Protected by struct _WsbmSlabCache::mutex
119     */
120
121    struct _WsbmSlabPool *slabPool;
122    uint32_t proposedPlacement;
123    struct _WsbmListHead timeoutHead;
124    struct _WsbmListHead head;
125    struct timeval timeFreed;
126};
127
128struct _WsbmSlab
129{
130    struct _WsbmListHead head;
131    struct _WsbmListHead freeBuffers;
132    uint32_t numBuffers;
133    uint32_t numFree;
134    struct _WsbmSlabBuffer *buffers;
135    struct _WsbmSlabSizeHeader *header;
136    struct _WsbmSlabKernelBO *kbo;
137};
138
139struct _WsbmSlabSizeHeader
140{
141    /*
142     * Constant at creation.
143     */
144    struct _WsbmSlabPool *slabPool;
145    uint32_t bufSize;
146
147    /*
148     * Protected by this::mutex
149     */
150
151    struct _WsbmListHead slabs;
152    struct _WsbmListHead freeSlabs;
153    struct _WsbmListHead delayedBuffers;
154    uint32_t numDelayed;
155    struct _WsbmMutex mutex;
156};
157
158struct _WsbmSlabCache
159{
160    struct timeval slabTimeout;
161    struct timeval checkInterval;
162    struct timeval nextCheck;
163    struct _WsbmListHead timeoutList;
164    struct _WsbmListHead unCached;
165    struct _WsbmListHead cached;
166    struct _WsbmMutex mutex;
167};
168
169struct _WsbmSlabPool
170{
171    struct _WsbmBufferPool pool;
172
173    /*
174     * The data of this structure remains constant after
175     * initialization and thus needs no mutex protection.
176     */
177
178    unsigned int devOffset;
179    struct _WsbmSlabCache *cache;
180    uint32_t proposedPlacement;
181    uint32_t validMask;
182    uint32_t *bucketSizes;
183    uint32_t numBuckets;
184    uint32_t pageSize;
185    int pageAlignment;
186    int maxSlabSize;
187    int desiredNumBuffers;
188    struct _WsbmSlabSizeHeader *headers;
189};
190
191static inline struct _WsbmSlabPool *
192slabPoolFromPool(struct _WsbmBufferPool *pool)
193{
194    return containerOf(pool, struct _WsbmSlabPool, pool);
195}
196
197static inline struct _WsbmSlabPool *
198slabPoolFromBuf(struct _WsbmSlabBuffer *sBuf)
199{
200    return slabPoolFromPool(sBuf->storage.pool);
201}
202
203static inline struct _WsbmSlabBuffer *
204slabBuffer(struct _WsbmBufStorage *buf)
205{
206    return containerOf(buf, struct _WsbmSlabBuffer, storage);
207}
208
209/*
210 * FIXME: Perhaps arrange timeout slabs in size buckets for fast
211 * retreival??
212 */
213
214static inline int
215wsbmTimeAfterEq(struct timeval *arg1, struct timeval *arg2)
216{
217    return ((arg1->tv_sec > arg2->tv_sec) ||
218	    ((arg1->tv_sec == arg2->tv_sec) &&
219	     (arg1->tv_usec > arg2->tv_usec)));
220}
221
222static inline void
223wsbmTimeAdd(struct timeval *arg, struct timeval *add)
224{
225    unsigned int sec;
226
227    arg->tv_sec += add->tv_sec;
228    arg->tv_usec += add->tv_usec;
229    sec = arg->tv_usec / 1000000;
230    arg->tv_sec += sec;
231    arg->tv_usec -= sec * 1000000;
232}
233
234static void
235wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo)
236{
237    struct ttm_pl_reference_req arg;
238    struct _WsbmSlabPool *slabPool;
239
240    if (!kbo)
241	return;
242
243    slabPool = kbo->slabPool;
244    arg.handle = kbo->kBuf.handle;
245    (void)munmap(kbo->virtual, kbo->actualSize);
246    (void)drmCommandWrite(slabPool->pool.fd,
247			  slabPool->devOffset + TTM_PL_UNREF, &arg,
248			  sizeof(arg));
249    free(kbo);
250}
251
252static void
253wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache *cache, struct timeval *time)
254{
255    struct _WsbmListHead *list, *next;
256    struct _WsbmSlabKernelBO *kbo;
257
258    if (!wsbmTimeAfterEq(time, &cache->nextCheck))
259	return;
260
261    WSBMLISTFOREACHSAFE(list, next, &cache->timeoutList) {
262	kbo = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, timeoutHead);
263
264	if (!wsbmTimeAfterEq(time, &kbo->timeFreed))
265	    break;
266
267	WSBMLISTDELINIT(&kbo->timeoutHead);
268	WSBMLISTDELINIT(&kbo->head);
269	wsbmFreeKernelBO(kbo);
270    }
271
272    cache->nextCheck = *time;
273    wsbmTimeAdd(&cache->nextCheck, &cache->checkInterval);
274}
275
276/*
277 * Add a _SlabKernelBO to the free slab manager.
278 * This means that it is available for reuse, but if it's not
279 * reused in a while, it will be freed.
280 */
281
282static void
283wsbmSetKernelBOFree(struct _WsbmSlabCache *cache,
284		    struct _WsbmSlabKernelBO *kbo)
285{
286    struct timeval time;
287    struct timeval timeFreed;
288
289    gettimeofday(&time, NULL);
290    timeFreed = time;
291    WSBM_MUTEX_LOCK(&cache->mutex);
292    wsbmTimeAdd(&timeFreed, &cache->slabTimeout);
293    kbo->timeFreed = timeFreed;
294
295    if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED)
296	WSBMLISTADD(&kbo->head, &cache->cached);
297    else
298	WSBMLISTADD(&kbo->head, &cache->unCached);
299
300    WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList);
301    wsbmFreeTimeoutKBOsLocked(cache, &time);
302
303    WSBM_MUTEX_UNLOCK(&cache->mutex);
304}
305
306/*
307 * Get a _SlabKernelBO for us to use as storage for a slab.
308 */
309
310static struct _WsbmSlabKernelBO *
311wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header)
312{
313    struct _WsbmSlabPool *slabPool = header->slabPool;
314    struct _WsbmSlabCache *cache = slabPool->cache;
315    struct _WsbmListHead *list, *head;
316    uint32_t size = header->bufSize * slabPool->desiredNumBuffers;
317    struct _WsbmSlabKernelBO *kbo;
318    struct _WsbmSlabKernelBO *kboTmp;
319    int ret;
320
321    /*
322     * FIXME: We should perhaps allow some variation in slabsize in order
323     * to efficiently reuse slabs.
324     */
325
326    size = (size <= (uint32_t) slabPool->maxSlabSize) ? size : (uint32_t) slabPool->maxSlabSize;
327    if (size < header->bufSize)
328	size = header->bufSize;
329    size = (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1);
330    WSBM_MUTEX_LOCK(&cache->mutex);
331
332    kbo = NULL;
333
334  retry:
335    head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ?
336	&cache->cached : &cache->unCached;
337
338    WSBMLISTFOREACH(list, head) {
339	kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head);
340
341	if ((kboTmp->actualSize == size) &&
342	    (slabPool->pageAlignment == 0 ||
343	     (kboTmp->pageAlignment % slabPool->pageAlignment) == 0)) {
344
345	    if (!kbo)
346		kbo = kboTmp;
347
348	    if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0)
349		break;
350
351	}
352    }
353
354    if (kbo) {
355	WSBMLISTDELINIT(&kbo->head);
356	WSBMLISTDELINIT(&kbo->timeoutHead);
357    }
358
359    WSBM_MUTEX_UNLOCK(&cache->mutex);
360
361    if (kbo) {
362	uint32_t new_mask =
363	    kbo->proposedPlacement ^ slabPool->proposedPlacement;
364
365	ret = 0;
366	if (new_mask) {
367	    union ttm_pl_setstatus_arg arg;
368	    struct ttm_pl_setstatus_req *req = &arg.req;
369	    struct ttm_pl_rep *rep = &arg.rep;
370
371	    req->handle = kbo->kBuf.handle;
372	    req->set_placement = slabPool->proposedPlacement & new_mask;
373	    req->clr_placement = ~slabPool->proposedPlacement & new_mask;
374	    DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
375				       slabPool->devOffset + TTM_PL_SETSTATUS,
376				       arg, ret);
377	    if (ret == 0) {
378		kbo->kBuf.gpuOffset = rep->gpu_offset;
379		kbo->kBuf.placement = rep->placement;
380	    }
381	    kbo->proposedPlacement = slabPool->proposedPlacement;
382	}
383
384	if (ret == 0)
385	    return kbo;
386
387	wsbmFreeKernelBO(kbo);
388	kbo = NULL;
389	goto retry;
390    }
391
392    kbo = calloc(1, sizeof(*kbo));
393    if (!kbo)
394	return NULL;
395
396    {
397	union ttm_pl_create_arg arg;
398
399	kbo->slabPool = slabPool;
400	WSBMINITLISTHEAD(&kbo->head);
401	WSBMINITLISTHEAD(&kbo->timeoutHead);
402
403	arg.req.size = size;
404	arg.req.placement = slabPool->proposedPlacement;
405	arg.req.page_alignment = slabPool->pageAlignment;
406
407	DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
408				   slabPool->devOffset + TTM_PL_CREATE,
409				   arg, ret);
410	if (ret)
411	    goto out_err0;
412
413	kbo->kBuf.gpuOffset = arg.rep.gpu_offset;
414	kbo->kBuf.placement = arg.rep.placement;
415	kbo->kBuf.handle = arg.rep.handle;
416
417	kbo->actualSize = arg.rep.bo_size;
418	kbo->mapHandle = arg.rep.map_handle;
419	kbo->proposedPlacement = slabPool->proposedPlacement;
420    }
421
422    kbo->virtual = mmap(0, kbo->actualSize,
423			PROT_READ | PROT_WRITE, MAP_SHARED,
424			slabPool->pool.fd, kbo->mapHandle);
425
426    if (kbo->virtual == MAP_FAILED) {
427	ret = -errno;
428	goto out_err1;
429    }
430
431    return kbo;
432
433  out_err1:
434    {
435	struct ttm_pl_reference_req arg = {.handle = kbo->kBuf.handle };
436
437	(void)drmCommandWrite(slabPool->pool.fd,
438			      slabPool->devOffset + TTM_PL_UNREF,
439			      &arg, sizeof(arg));
440    }
441  out_err0:
442    free(kbo);
443    return NULL;
444}
445
446static int
447wsbmAllocSlab(struct _WsbmSlabSizeHeader *header)
448{
449    struct _WsbmSlab *slab;
450    struct _WsbmSlabBuffer *sBuf;
451    uint32_t numBuffers;
452    uint32_t ret;
453    uint32_t i;
454
455    slab = calloc(1, sizeof(*slab));
456    if (!slab)
457	return -ENOMEM;
458
459    slab->kbo = wsbmAllocKernelBO(header);
460    if (!slab->kbo) {
461	ret = -ENOMEM;
462	goto out_err0;
463    }
464
465    numBuffers = slab->kbo->actualSize / header->bufSize;
466    if (!numBuffers) {
467        ret = -ENOMEM;
468        goto out_err1;
469    }
470
471    slab->buffers = calloc(numBuffers, sizeof(*slab->buffers));
472    if (!slab->buffers) {
473	ret = -ENOMEM;
474	goto out_err1;
475    }
476
477    WSBMINITLISTHEAD(&slab->head);
478    WSBMINITLISTHEAD(&slab->freeBuffers);
479    slab->numBuffers = numBuffers;
480    slab->numFree = 0;
481    slab->header = header;
482
483    sBuf = slab->buffers;
484    for (i = 0; i < numBuffers; ++i) {
485	ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool);
486	if (ret)
487	    goto out_err2;
488	sBuf->parent = slab;
489	sBuf->start = i * header->bufSize;
490	sBuf->virtual = (void *)((uint8_t *) slab->kbo->virtual +
491				 sBuf->start);
492	wsbmAtomicSet(&sBuf->writers, 0);
493	sBuf->isSlabBuffer = 1;
494	WSBM_COND_INIT(&sBuf->event);
495	WSBMLISTADDTAIL(&sBuf->head, &slab->freeBuffers);
496	slab->numFree++;
497	sBuf++;
498    }
499
500    WSBMLISTADDTAIL(&slab->head, &header->slabs);
501
502    return 0;
503
504  out_err2:
505    sBuf = slab->buffers;
506    for (i = 0; i < numBuffers; ++i) {
507	if (sBuf->parent == slab) {
508	    WSBM_COND_FREE(&sBuf->event);
509	    wsbmBufStorageTakedown(&sBuf->storage);
510	}
511	sBuf++;
512    }
513    free(slab->buffers);
514  out_err1:
515    wsbmSetKernelBOFree(header->slabPool->cache, slab->kbo);
516  out_err0:
517    free(slab);
518    return ret;
519}
520
521/*
522 * Delete a buffer from the slab header delayed list and put
523 * it on the slab free list.
524 */
525
526static void
527wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer *buf)
528{
529    struct _WsbmSlab *slab = buf->parent;
530    struct _WsbmSlabSizeHeader *header = slab->header;
531    struct _WsbmListHead *list = &buf->head;
532
533    WSBMLISTDEL(list);
534    WSBMLISTADDTAIL(list, &slab->freeBuffers);
535    slab->numFree++;
536
537    if (slab->head.next == &slab->head)
538	WSBMLISTADDTAIL(&slab->head, &header->slabs);
539
540    if (slab->numFree == slab->numBuffers) {
541	list = &slab->head;
542	WSBMLISTDEL(list);
543	WSBMLISTADDTAIL(list, &header->freeSlabs);
544    }
545
546    if (header->slabs.next == &header->slabs ||
547	slab->numFree != slab->numBuffers) {
548
549	struct _WsbmListHead *next;
550	struct _WsbmSlabCache *cache = header->slabPool->cache;
551
552	WSBMLISTFOREACHSAFE(list, next, &header->freeSlabs) {
553	    uint32_t i;
554	    struct _WsbmSlabBuffer *sBuf;
555
556	    slab = WSBMLISTENTRY(list, struct _WsbmSlab, head);
557
558	    WSBMLISTDELINIT(list);
559
560	    sBuf = slab->buffers;
561	    for (i = 0; i < slab->numBuffers; ++i) {
562		if (sBuf->parent == slab) {
563		    WSBM_COND_FREE(&sBuf->event);
564		    wsbmBufStorageTakedown(&sBuf->storage);
565		}
566		sBuf++;
567	    }
568	    wsbmSetKernelBOFree(cache, slab->kbo);
569	    free(slab->buffers);
570	    free(slab);
571	}
572    }
573}
574
575static void
576wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader *header, int wait)
577{
578  struct _WsbmListHead *list, *prev, *first, *head;
579    struct _WsbmSlabBuffer *sBuf;
580    struct _WsbmSlab *slab;
581    int firstWasSignaled = 1;
582    int signaled;
583    uint32_t i;
584    int ret;
585
586    /*
587     * Rerun the freeing test if the youngest tested buffer
588     * was signaled, since there might be more idle buffers
589     * in the delay list.
590     */
591
592    while (firstWasSignaled) {
593	firstWasSignaled = 0;
594	signaled = 0;
595	first = header->delayedBuffers.next;
596
597	/* Only examine the oldest 1/3 of delayed buffers:
598	 */
599	if (header->numDelayed > 3) {
600	    for (i = 0; i < header->numDelayed; i += 3) {
601		first = first->next;
602	    }
603	}
604
605	/*
606	 * No need to take the buffer mutex for each buffer we loop
607	 * through since we're currently the only user.
608	 */
609
610	head = first->next;
611	WSBMLISTFOREACHPREVSAFE(list, prev, head) {
612
613	    if (list == &header->delayedBuffers)
614		break;
615
616	    sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
617
618	    slab = sBuf->parent;
619
620	    if (!signaled) {
621		if (wait) {
622		    ret = wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);
623		    if (ret)
624			break;
625		    signaled = 1;
626		    wait = 0;
627		} else {
628		    signaled =
629			wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);
630#ifdef DEBUG_FENCESIGNALED
631		    fencesignaled++;
632#endif
633		}
634		if (signaled) {
635		    if (list == first)
636			firstWasSignaled = 1;
637		    wsbmFenceUnreference(&sBuf->fence);
638		    header->numDelayed--;
639		    wsbmSlabFreeBufferLocked(sBuf);
640		} else
641		    break;
642	    } else if (wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {
643		wsbmFenceUnreference(&sBuf->fence);
644		header->numDelayed--;
645		wsbmSlabFreeBufferLocked(sBuf);
646	    }
647	}
648    }
649}
650
651static struct _WsbmSlabBuffer *
652wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header)
653{
654    static struct _WsbmSlabBuffer *buf;
655    struct _WsbmSlab *slab;
656    struct _WsbmListHead *list;
657    int count = WSBM_SLABPOOL_ALLOC_RETRIES;
658
659    WSBM_MUTEX_LOCK(&header->mutex);
660    while (header->slabs.next == &header->slabs && count > 0) {
661	wsbmSlabCheckFreeLocked(header, 0);
662	if (header->slabs.next != &header->slabs)
663	    break;
664
665	WSBM_MUTEX_UNLOCK(&header->mutex);
666	if (count != WSBM_SLABPOOL_ALLOC_RETRIES)
667	    usleep(1000);
668	WSBM_MUTEX_LOCK(&header->mutex);
669	(void)wsbmAllocSlab(header);
670	count--;
671    }
672
673    list = header->slabs.next;
674    if (list == &header->slabs) {
675	WSBM_MUTEX_UNLOCK(&header->mutex);
676	return NULL;
677    }
678    slab = WSBMLISTENTRY(list, struct _WsbmSlab, head);
679    if (--slab->numFree == 0)
680	WSBMLISTDELINIT(list);
681
682    list = slab->freeBuffers.next;
683    WSBMLISTDELINIT(list);
684
685    WSBM_MUTEX_UNLOCK(&header->mutex);
686    buf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
687
688    buf->storage.destroyContainer = NULL;
689
690#ifdef DEBUG_FENCESIGNALED
691    createbuffer++;
692#endif
693    return buf;
694}
695
696static struct _WsbmBufStorage *
697pool_create(struct _WsbmBufferPool *pool, unsigned long size,
698	    uint32_t placement, unsigned alignment)
699{
700    struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool);
701    struct _WsbmSlabSizeHeader *header;
702    struct _WsbmSlabBuffer *sBuf;
703    uint32_t i;
704    int ret;
705
706    /*
707     * FIXME: Check for compatibility.
708     */
709
710    header = slabPool->headers;
711    for (i = 0; i < slabPool->numBuckets; ++i) {
712	if (header->bufSize >= size)
713	    break;
714	header++;
715    }
716
717    if (i < slabPool->numBuckets) {
718	sBuf = wsbmSlabAllocBuffer(header);
719	return ((sBuf) ? &sBuf->storage : NULL);
720    }
721
722    /*
723     * Fall back to allocate a buffer object directly from DRM.
724     * and wrap it in a wsbmBO structure.
725     */
726
727    sBuf = calloc(1, sizeof(*sBuf));
728
729    if (!sBuf)
730	return NULL;
731
732    if (alignment) {
733	if ((alignment < slabPool->pageSize)
734	    && (slabPool->pageSize % alignment))
735	    goto out_err0;
736	if ((alignment > slabPool->pageSize)
737	    && (alignment % slabPool->pageSize))
738	    goto out_err0;
739    }
740
741    ret = wsbmBufStorageInit(&sBuf->storage, pool);
742    if (ret)
743	goto out_err0;
744
745    ret = WSBM_COND_INIT(&sBuf->event);
746    if (ret)
747	goto out_err1;
748
749    {
750	union ttm_pl_create_arg arg;
751
752	arg.req.size = size;
753	arg.req.placement = placement;
754	arg.req.page_alignment = alignment / slabPool->pageSize;
755
756	DRMRESTARTCOMMANDWRITEREAD(pool->fd,
757				   slabPool->devOffset + TTM_PL_CREATE,
758				   arg, ret);
759
760	if (ret)
761	    goto out_err2;
762
763	sBuf->kBuf.gpuOffset = arg.rep.gpu_offset;
764	sBuf->kBuf.placement = arg.rep.placement;
765	sBuf->kBuf.handle = arg.rep.handle;
766	sBuf->mapHandle = arg.rep.map_handle;
767	sBuf->requestedSize = size;
768
769	sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED,
770			     pool->fd, sBuf->mapHandle);
771
772	if (sBuf->virtual == MAP_FAILED)
773	    goto out_err3;
774    }
775
776    wsbmAtomicSet(&sBuf->writers, 0);
777    return &sBuf->storage;
778  out_err3:
779    {
780	struct ttm_pl_reference_req arg;
781
782	arg.handle = sBuf->kBuf.handle;
783	(void)drmCommandWriteRead(pool->fd,
784				  slabPool->devOffset + TTM_PL_UNREF,
785				  &arg, sizeof(arg));
786    }
787  out_err2:
788    WSBM_COND_FREE(&sBuf->event);
789  out_err1:
790    wsbmBufStorageTakedown(&sBuf->storage);
791  out_err0:
792    free(sBuf);
793    return NULL;
794}
795
796static void
797pool_destroy(struct _WsbmBufStorage **p_buf)
798{
799    struct _WsbmBufStorage *buf = *p_buf;
800    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
801    struct _WsbmSlab *slab;
802    struct _WsbmSlabSizeHeader *header;
803
804    *p_buf = NULL;
805
806    if (!sBuf->isSlabBuffer) {
807	struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf);
808	struct ttm_pl_reference_req arg;
809
810	if (sBuf->virtual != NULL) {
811	    (void)munmap(sBuf->virtual, sBuf->requestedSize);
812	    sBuf->virtual = NULL;
813	}
814
815	arg.handle = sBuf->kBuf.handle;
816	(void)drmCommandWrite(slabPool->pool.fd,
817			      slabPool->devOffset + TTM_PL_UNREF,
818			      &arg, sizeof(arg));
819
820	WSBM_COND_FREE(&sBuf->event);
821	wsbmBufStorageTakedown(&sBuf->storage);
822	free(sBuf);
823	return;
824    }
825
826    slab = sBuf->parent;
827    header = slab->header;
828
829    /*
830     * No need to take the buffer mutex below since we're the only user.
831     */
832
833    WSBM_MUTEX_LOCK(&header->mutex);
834    sBuf->unFenced = 0;
835    wsbmAtomicSet(&sBuf->writers, 0);
836    wsbmAtomicSet(&sBuf->storage.refCount, 1);
837
838    if (sBuf->fence && !wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {
839	WSBMLISTADDTAIL(&sBuf->head, &header->delayedBuffers);
840	header->numDelayed++;
841    } else {
842	if (sBuf->fence)
843	    wsbmFenceUnreference(&sBuf->fence);
844	wsbmSlabFreeBufferLocked(sBuf);
845    }
846
847    WSBM_MUTEX_UNLOCK(&header->mutex);
848}
849
850static void
851waitIdleLocked(struct _WsbmSlabBuffer *sBuf, int lazy)
852{
853    struct _WsbmBufStorage *storage = &sBuf->storage;
854
855    while (sBuf->unFenced || sBuf->fence != NULL) {
856
857	if (sBuf->unFenced)
858	    WSBM_COND_WAIT(&sBuf->event, &storage->mutex);
859
860	if (sBuf->fence != NULL) {
861	    if (!wsbmFenceSignaled(sBuf->fence, sBuf->fenceType)) {
862		struct _WsbmFenceObject *fence =
863		    wsbmFenceReference(sBuf->fence);
864
865		WSBM_MUTEX_UNLOCK(&storage->mutex);
866		(void)wsbmFenceFinish(fence, sBuf->fenceType, lazy);
867		WSBM_MUTEX_LOCK(&storage->mutex);
868		if (sBuf->fence == fence)
869		    wsbmFenceUnreference(&sBuf->fence);
870
871		wsbmFenceUnreference(&fence);
872	    } else {
873		wsbmFenceUnreference(&sBuf->fence);
874	    }
875	}
876    }
877}
878
879static int
880pool_waitIdle(struct _WsbmBufStorage *buf, int lazy)
881{
882    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
883
884    WSBM_MUTEX_LOCK(&buf->mutex);
885    waitIdleLocked(sBuf, lazy);
886    WSBM_MUTEX_UNLOCK(&buf->mutex);
887
888    return 0;
889}
890
891static int
892pool_map(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)), void **virtual)
893{
894    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
895
896    *virtual = sBuf->virtual;
897
898    return 0;
899}
900
901static void
902pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)))
903{
904    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
905
906    if (wsbmAtomicDecZero(&sBuf->writers))
907	WSBM_COND_BROADCAST(&sBuf->event);
908}
909
910static int
911pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode)
912{
913    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
914    int ret = 0;
915
916    WSBM_MUTEX_LOCK(&buf->mutex);
917    if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) {
918	int signaled;
919
920	if (sBuf->unFenced) {
921	    ret = -EBUSY;
922	    goto out_unlock;
923	}
924
925	if (sBuf->isSlabBuffer)
926	    signaled = (sBuf->fence == NULL) ||
927		wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType);
928	else
929	    signaled = (sBuf->fence == NULL) ||
930		wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);
931
932	ret = 0;
933	if (signaled) {
934	    wsbmFenceUnreference(&sBuf->fence);
935	    wsbmAtomicInc(&sBuf->writers);
936	} else
937	    ret = -EBUSY;
938	goto out_unlock;
939    }
940    waitIdleLocked(sBuf, 0);
941    wsbmAtomicInc(&sBuf->writers);
942  out_unlock:
943    WSBM_MUTEX_UNLOCK(&buf->mutex);
944    return ret;
945}
946
947static void
948pool_unmap(struct _WsbmBufStorage *buf __attribute__ ((unused)))
949{
950    ;
951}
952
953static unsigned long
954pool_poolOffset(struct _WsbmBufStorage *buf)
955{
956    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
957
958    return sBuf->start;
959}
960
961static unsigned long
962pool_size(struct _WsbmBufStorage *buf)
963{
964    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
965
966    if (!sBuf->isSlabBuffer)
967	return sBuf->requestedSize;
968
969    return sBuf->parent->header->bufSize;
970}
971
972static struct _WsbmKernelBuf *
973pool_kernel(struct _WsbmBufStorage *buf)
974{
975    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
976
977    return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf;
978}
979
980static unsigned long
981pool_offset(struct _WsbmBufStorage *buf)
982{
983    return pool_kernel(buf)->gpuOffset + pool_poolOffset(buf);
984}
985
986static void
987pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence)
988{
989    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
990    struct _WsbmKernelBuf *kBuf;
991
992    WSBM_MUTEX_LOCK(&buf->mutex);
993    if (sBuf->fence)
994	wsbmFenceUnreference(&sBuf->fence);
995
996    kBuf = pool_kernel(buf);
997    sBuf->fenceType = kBuf->fence_type_mask;
998    if (!wsbmFenceSignaledCached(fence, sBuf->fenceType))
999	sBuf->fence = wsbmFenceReference(fence);
1000
1001    sBuf->unFenced = 0;
1002    WSBM_COND_BROADCAST(&sBuf->event);
1003    WSBM_MUTEX_UNLOCK(&buf->mutex);
1004}
1005
1006static int
1007pool_validate(struct _WsbmBufStorage *buf,
1008        uint64_t set_flags __attribute__ ((unused)), uint64_t clr_flags __attribute__ ((unused)))
1009{
1010    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
1011
1012    WSBM_MUTEX_LOCK(&buf->mutex);
1013    while (wsbmAtomicRead(&sBuf->writers) != 0) {
1014	WSBM_COND_WAIT(&sBuf->event, &buf->mutex);
1015    }
1016
1017    sBuf->unFenced = 1;
1018    WSBM_MUTEX_UNLOCK(&buf->mutex);
1019    return 0;
1020}
1021
1022static void
1023pool_unvalidate(struct _WsbmBufStorage *buf)
1024{
1025    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
1026
1027    WSBM_MUTEX_LOCK(&buf->mutex);
1028    if (sBuf->unFenced) {
1029	sBuf->unFenced = 0;
1030	WSBM_COND_BROADCAST(&sBuf->event);
1031    }
1032    WSBM_MUTEX_UNLOCK(&buf->mutex);
1033}
1034
1035struct _WsbmSlabCache *
1036wsbmSlabCacheInit(uint32_t checkIntervalMsec, uint32_t slabTimeoutMsec)
1037{
1038    struct _WsbmSlabCache *tmp;
1039
1040    tmp = calloc(1, sizeof(*tmp));
1041    if (!tmp)
1042	return NULL;
1043
1044    WSBM_MUTEX_INIT(&tmp->mutex);
1045    WSBM_MUTEX_LOCK(&tmp->mutex);
1046    tmp->slabTimeout.tv_usec = slabTimeoutMsec * 1000;
1047    tmp->slabTimeout.tv_sec = tmp->slabTimeout.tv_usec / 1000000;
1048    tmp->slabTimeout.tv_usec -= tmp->slabTimeout.tv_sec * 1000000;
1049
1050    tmp->checkInterval.tv_usec = checkIntervalMsec * 1000;
1051    tmp->checkInterval.tv_sec = tmp->checkInterval.tv_usec / 1000000;
1052    tmp->checkInterval.tv_usec -= tmp->checkInterval.tv_sec * 1000000;
1053
1054    gettimeofday(&tmp->nextCheck, NULL);
1055    wsbmTimeAdd(&tmp->nextCheck, &tmp->checkInterval);
1056    WSBMINITLISTHEAD(&tmp->timeoutList);
1057    WSBMINITLISTHEAD(&tmp->unCached);
1058    WSBMINITLISTHEAD(&tmp->cached);
1059    WSBM_MUTEX_UNLOCK(&tmp->mutex);
1060
1061    return tmp;
1062}
1063
1064void
1065wsbmSlabCacheFinish(struct _WsbmSlabCache *cache)
1066{
1067    struct timeval time;
1068
1069    time = cache->nextCheck;
1070    WSBM_MUTEX_LOCK(&cache->mutex);
1071    wsbmTimeAdd(&time, &cache->checkInterval);
1072    wsbmFreeTimeoutKBOsLocked(cache, &time);
1073    WSBM_MUTEX_UNLOCK(&cache->mutex);
1074
1075    assert(cache->timeoutList.next == &cache->timeoutList);
1076    assert(cache->unCached.next == &cache->unCached);
1077    assert(cache->cached.next == &cache->cached);
1078
1079    WSBM_MUTEX_FREE(&cache->mutex);
1080    free(cache);
1081}
1082
1083static void
1084wsbmInitSizeHeader(struct _WsbmSlabPool *slabPool, uint32_t size,
1085		   struct _WsbmSlabSizeHeader *header)
1086{
1087    WSBM_MUTEX_INIT(&header->mutex);
1088    WSBM_MUTEX_LOCK(&header->mutex);
1089
1090    WSBMINITLISTHEAD(&header->slabs);
1091    WSBMINITLISTHEAD(&header->freeSlabs);
1092    WSBMINITLISTHEAD(&header->delayedBuffers);
1093
1094    header->numDelayed = 0;
1095    header->slabPool = slabPool;
1096    header->bufSize = size;
1097
1098    WSBM_MUTEX_UNLOCK(&header->mutex);
1099}
1100
1101static void
1102wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader *header)
1103{
1104    struct _WsbmListHead *list, *next;
1105    struct _WsbmSlabBuffer *sBuf;
1106
1107    WSBM_MUTEX_LOCK(&header->mutex);
1108    WSBMLISTFOREACHSAFE(list, next, &header->delayedBuffers) {
1109	sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
1110
1111	if (sBuf->fence) {
1112	    (void)wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);
1113	    wsbmFenceUnreference(&sBuf->fence);
1114	}
1115	header->numDelayed--;
1116	wsbmSlabFreeBufferLocked(sBuf);
1117    }
1118    WSBM_MUTEX_UNLOCK(&header->mutex);
1119    WSBM_MUTEX_FREE(&header->mutex);
1120}
1121
1122static void
1123pool_takedown(struct _WsbmBufferPool *pool)
1124{
1125    struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool);
1126    unsigned int i;
1127
1128    for (i = 0; i < slabPool->numBuckets; ++i) {
1129	wsbmFinishSizeHeader(&slabPool->headers[i]);
1130    }
1131
1132    free(slabPool->headers);
1133    free(slabPool->bucketSizes);
1134    free(slabPool);
1135}
1136
1137struct _WsbmBufferPool *
1138wsbmSlabPoolInit(int fd,
1139		 uint32_t devOffset,
1140		 uint32_t placement,
1141		 uint32_t validMask,
1142		 uint32_t smallestSize,
1143		 uint32_t numSizes,
1144		 uint32_t desiredNumBuffers,
1145		 uint32_t maxSlabSize,
1146		 uint32_t pageAlignment, struct _WsbmSlabCache *cache)
1147{
1148    struct _WsbmBufferPool *pool;
1149    struct _WsbmSlabPool *slabPool;
1150    uint32_t i;
1151
1152    slabPool = calloc(1, sizeof(*slabPool));
1153    if (!slabPool)
1154	return NULL;
1155
1156    pool = &slabPool->pool;
1157
1158    slabPool->bucketSizes = calloc(numSizes, sizeof(*slabPool->bucketSizes));
1159    if (!slabPool->bucketSizes)
1160	goto out_err0;
1161
1162    slabPool->headers = calloc(numSizes, sizeof(*slabPool->headers));
1163    if (!slabPool->headers)
1164	goto out_err1;
1165
1166    slabPool->devOffset = devOffset;
1167    slabPool->cache = cache;
1168    slabPool->proposedPlacement = placement;
1169    slabPool->validMask = validMask;
1170    slabPool->numBuckets = numSizes;
1171    slabPool->pageSize = getpagesize();
1172    slabPool->pageAlignment = pageAlignment;
1173    slabPool->maxSlabSize = maxSlabSize;
1174    slabPool->desiredNumBuffers = desiredNumBuffers;
1175
1176    for (i = 0; i < slabPool->numBuckets; ++i) {
1177	slabPool->bucketSizes[i] = (smallestSize << i);
1178	wsbmInitSizeHeader(slabPool, slabPool->bucketSizes[i],
1179			   &slabPool->headers[i]);
1180    }
1181
1182    pool->fd = fd;
1183    pool->map = &pool_map;
1184    pool->unmap = &pool_unmap;
1185    pool->destroy = &pool_destroy;
1186    pool->offset = &pool_offset;
1187    pool->poolOffset = &pool_poolOffset;
1188    pool->size = &pool_size;
1189    pool->create = &pool_create;
1190    pool->fence = &pool_fence;
1191    pool->kernel = &pool_kernel;
1192    pool->validate = &pool_validate;
1193    pool->unvalidate = &pool_unvalidate;
1194    pool->waitIdle = &pool_waitIdle;
1195    pool->takeDown = &pool_takedown;
1196    pool->releasefromcpu = &pool_releaseFromCpu;
1197    pool->syncforcpu = &pool_syncForCpu;
1198
1199    return pool;
1200
1201  out_err1:
1202    free(slabPool->bucketSizes);
1203  out_err0:
1204    free(slabPool);
1205
1206    return NULL;
1207}
1208