1/**************************************************************************
2 *
3 * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA
4 * All Rights Reserved.
5 * Copyright 2009 VMware, Inc., Palo Alto, CA., USA
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
23 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
24 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
25 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
26 * USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
28 **************************************************************************/
29/*
30 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
31 */
32
33#ifdef HAVE_CONFIG_H
34#include "config.h"
35#endif
36
37#include <drm/psb_ttm_placement_user.h>
38#include <stdint.h>
39#include <sys/time.h>
40#include <errno.h>
41#include <unistd.h>
42#include <assert.h>
43#include <string.h>
44#include <sys/mman.h>
45#include <xf86drm.h>
46#include "wsbm_pool.h"
47#include "wsbm_fencemgr.h"
48#include "wsbm_priv.h"
49#include "wsbm_manager.h"
50
51#define WSBM_SLABPOOL_ALLOC_RETRIES 100
52#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret)			\
53	do {								\
54		(_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \
55	} while ((_ret) == -EAGAIN || (_ret) == -ERESTART);		\
56
57#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret)		\
58	do {								\
59		(_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \
60	} while ((_ret) == -EAGAIN || (_ret) == -ERESTART);		\
61
62
63#ifdef DEBUG_FENCESIGNALED
64static int createbuffer = 0;
65static int fencesignaled = 0;
66#endif
67
68struct _WsbmSlab;
69
70struct _WsbmSlabBuffer
71{
72    struct _WsbmKernelBuf kBuf;
73    struct _WsbmBufStorage storage;
74    struct _WsbmCond event;
75
76    /*
77     * Remains constant after creation.
78     */
79
80    int isSlabBuffer;
81    struct _WsbmSlab *parent;
82    uint32_t start;
83    void *virtual;
84    unsigned long requestedSize;
85    uint64_t mapHandle;
86
87    /*
88     * Protected by struct _WsbmSlabSizeHeader::mutex
89     */
90
91    struct _WsbmListHead head;
92
93    /*
94     * Protected by this::mutex
95     */
96
97    struct _WsbmFenceObject *fence;
98    uint32_t fenceType;
99    struct _WsbmAtomic writers;	       /* (Only upping) */
100    int unFenced;
101};
102
103struct _WsbmSlabPool;
104struct _WsbmSlabKernelBO
105{
106
107    /*
108     * Constant at creation
109     */
110
111    struct _WsbmKernelBuf kBuf;
112    uint32_t pageAlignment;
113    void *virtual;
114    unsigned long actualSize;
115    uint64_t mapHandle;
116
117    /*
118     * Protected by struct _WsbmSlabCache::mutex
119     */
120
121    struct _WsbmSlabPool *slabPool;
122    uint32_t proposedPlacement;
123    struct _WsbmListHead timeoutHead;
124    struct _WsbmListHead head;
125    struct timeval timeFreed;
126};
127
128struct _WsbmSlab
129{
130    struct _WsbmListHead head;
131    struct _WsbmListHead freeBuffers;
132    uint32_t numBuffers;
133    uint32_t numFree;
134    struct _WsbmSlabBuffer *buffers;
135    struct _WsbmSlabSizeHeader *header;
136    struct _WsbmSlabKernelBO *kbo;
137};
138
139struct _WsbmSlabSizeHeader
140{
141    /*
142     * Constant at creation.
143     */
144    struct _WsbmSlabPool *slabPool;
145    uint32_t bufSize;
146
147    /*
148     * Protected by this::mutex
149     */
150
151    struct _WsbmListHead slabs;
152    struct _WsbmListHead freeSlabs;
153    struct _WsbmListHead delayedBuffers;
154    uint32_t numDelayed;
155    struct _WsbmMutex mutex;
156};
157
158struct _WsbmSlabCache
159{
160    struct timeval slabTimeout;
161    struct timeval checkInterval;
162    struct timeval nextCheck;
163    struct _WsbmListHead timeoutList;
164    struct _WsbmListHead unCached;
165    struct _WsbmListHead cached;
166    struct _WsbmMutex mutex;
167};
168
169struct _WsbmSlabPool
170{
171    struct _WsbmBufferPool pool;
172
173    /*
174     * The data of this structure remains constant after
175     * initialization and thus needs no mutex protection.
176     */
177
178    unsigned int devOffset;
179    struct _WsbmSlabCache *cache;
180    uint32_t proposedPlacement;
181    uint32_t validMask;
182    uint32_t *bucketSizes;
183    uint32_t numBuckets;
184    uint32_t pageSize;
185    int pageAlignment;
186    int maxSlabSize;
187    int desiredNumBuffers;
188    struct _WsbmSlabSizeHeader *headers;
189};
190
191static inline struct _WsbmSlabPool *
192slabPoolFromPool(struct _WsbmBufferPool *pool)
193{
194    return containerOf(pool, struct _WsbmSlabPool, pool);
195}
196
197static inline struct _WsbmSlabPool *
198slabPoolFromBuf(struct _WsbmSlabBuffer *sBuf)
199{
200    return slabPoolFromPool(sBuf->storage.pool);
201}
202
203static inline struct _WsbmSlabBuffer *
204slabBuffer(struct _WsbmBufStorage *buf)
205{
206    return containerOf(buf, struct _WsbmSlabBuffer, storage);
207}
208
209/*
210 * FIXME: Perhaps arrange timeout slabs in size buckets for fast
211 * retreival??
212 */
213
214static inline int
215wsbmTimeAfterEq(struct timeval *arg1, struct timeval *arg2)
216{
217    return ((arg1->tv_sec > arg2->tv_sec) ||
218	    ((arg1->tv_sec == arg2->tv_sec) &&
219	     (arg1->tv_usec > arg2->tv_usec)));
220}
221
222static inline void
223wsbmTimeAdd(struct timeval *arg, struct timeval *add)
224{
225    unsigned int sec;
226
227    arg->tv_sec += add->tv_sec;
228    arg->tv_usec += add->tv_usec;
229    sec = arg->tv_usec / 1000000;
230    arg->tv_sec += sec;
231    arg->tv_usec -= sec * 1000000;
232}
233
234static void
235wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo)
236{
237    struct ttm_pl_reference_req arg;
238    struct _WsbmSlabPool *slabPool;
239
240    if (!kbo)
241	return;
242
243    slabPool = kbo->slabPool;
244    arg.handle = kbo->kBuf.handle;
245    (void)munmap(kbo->virtual, kbo->actualSize);
246    (void)drmCommandWrite(slabPool->pool.fd,
247			  slabPool->devOffset + TTM_PL_UNREF, &arg,
248			  sizeof(arg));
249    free(kbo);
250}
251
252static void
253wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache *cache, struct timeval *time)
254{
255    struct _WsbmListHead *list, *next;
256    struct _WsbmSlabKernelBO *kbo;
257
258    if (!wsbmTimeAfterEq(time, &cache->nextCheck))
259	return;
260
261    WSBMLISTFOREACHSAFE(list, next, &cache->timeoutList) {
262	kbo = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, timeoutHead);
263
264	if (!wsbmTimeAfterEq(time, &kbo->timeFreed))
265	    break;
266
267	WSBMLISTDELINIT(&kbo->timeoutHead);
268	WSBMLISTDELINIT(&kbo->head);
269	wsbmFreeKernelBO(kbo);
270    }
271
272    cache->nextCheck = *time;
273    wsbmTimeAdd(&cache->nextCheck, &cache->checkInterval);
274}
275
276/*
277 * Add a _SlabKernelBO to the free slab manager.
278 * This means that it is available for reuse, but if it's not
279 * reused in a while, it will be freed.
280 */
281
282static void
283wsbmSetKernelBOFree(struct _WsbmSlabCache *cache,
284		    struct _WsbmSlabKernelBO *kbo)
285{
286    struct timeval time;
287    struct timeval timeFreed;
288
289    gettimeofday(&time, NULL);
290    timeFreed = time;
291    WSBM_MUTEX_LOCK(&cache->mutex);
292    wsbmTimeAdd(&timeFreed, &cache->slabTimeout);
293    kbo->timeFreed = timeFreed;
294
295    if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED)
296	WSBMLISTADD(&kbo->head, &cache->cached);
297    else
298	WSBMLISTADD(&kbo->head, &cache->unCached);
299
300    WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList);
301    wsbmFreeTimeoutKBOsLocked(cache, &time);
302
303    WSBM_MUTEX_UNLOCK(&cache->mutex);
304}
305
306/*
307 * Get a _SlabKernelBO for us to use as storage for a slab.
308 */
309
310static struct _WsbmSlabKernelBO *
311wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header)
312{
313    struct _WsbmSlabPool *slabPool = header->slabPool;
314    struct _WsbmSlabCache *cache = slabPool->cache;
315    struct _WsbmListHead *list, *head;
316    uint32_t size = header->bufSize * slabPool->desiredNumBuffers;
317    struct _WsbmSlabKernelBO *kbo;
318    struct _WsbmSlabKernelBO *kboTmp;
319    int ret;
320
321    /*
322     * FIXME: We should perhaps allow some variation in slabsize in order
323     * to efficiently reuse slabs.
324     */
325
326    size = (size <= (uint32_t) slabPool->maxSlabSize) ? size : (uint32_t) slabPool->maxSlabSize;
327    if (size < header->bufSize)
328	size = header->bufSize;
329    size = (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1);
330    WSBM_MUTEX_LOCK(&cache->mutex);
331
332    kbo = NULL;
333
334  retry:
335    head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ?
336	&cache->cached : &cache->unCached;
337
338    WSBMLISTFOREACH(list, head) {
339	kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head);
340
341	if ((kboTmp->actualSize == size) &&
342	    (slabPool->pageAlignment == 0 ||
343	     (kboTmp->pageAlignment % slabPool->pageAlignment) == 0)) {
344
345	    if (!kbo)
346		kbo = kboTmp;
347
348	    if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0)
349		break;
350
351	}
352    }
353
354    if (kbo) {
355	WSBMLISTDELINIT(&kbo->head);
356	WSBMLISTDELINIT(&kbo->timeoutHead);
357    }
358
359    WSBM_MUTEX_UNLOCK(&cache->mutex);
360
361    if (kbo) {
362	uint32_t new_mask =
363	    kbo->proposedPlacement ^ slabPool->proposedPlacement;
364
365	ret = 0;
366	if (new_mask) {
367	    union ttm_pl_setstatus_arg arg;
368	    struct ttm_pl_setstatus_req *req = &arg.req;
369	    struct ttm_pl_rep *rep = &arg.rep;
370
371	    req->handle = kbo->kBuf.handle;
372	    req->set_placement = slabPool->proposedPlacement & new_mask;
373	    req->clr_placement = ~slabPool->proposedPlacement & new_mask;
374	    DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
375				       slabPool->devOffset + TTM_PL_SETSTATUS,
376				       arg, ret);
377	    if (ret == 0) {
378		kbo->kBuf.gpuOffset = rep->gpu_offset;
379		kbo->kBuf.placement = rep->placement;
380	    }
381	    kbo->proposedPlacement = slabPool->proposedPlacement;
382	}
383
384	if (ret == 0)
385	    return kbo;
386
387	wsbmFreeKernelBO(kbo);
388	kbo = NULL;
389	goto retry;
390    }
391
392    kbo = calloc(1, sizeof(*kbo));
393    if (!kbo)
394	return NULL;
395
396    {
397	union ttm_pl_create_arg arg;
398
399	kbo->slabPool = slabPool;
400	WSBMINITLISTHEAD(&kbo->head);
401	WSBMINITLISTHEAD(&kbo->timeoutHead);
402
403	arg.req.size = size;
404	arg.req.placement = slabPool->proposedPlacement;
405	arg.req.page_alignment = slabPool->pageAlignment;
406
407	DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
408				   slabPool->devOffset + TTM_PL_CREATE,
409				   arg, ret);
410	if (ret)
411	    goto out_err0;
412
413	kbo->kBuf.gpuOffset = arg.rep.gpu_offset;
414	kbo->kBuf.placement = arg.rep.placement;
415	kbo->kBuf.handle = arg.rep.handle;
416
417	kbo->actualSize = arg.rep.bo_size;
418	kbo->mapHandle = arg.rep.map_handle;
419	kbo->proposedPlacement = slabPool->proposedPlacement;
420    }
421
422    kbo->virtual = mmap(0, kbo->actualSize,
423			PROT_READ | PROT_WRITE, MAP_SHARED,
424			slabPool->pool.fd, kbo->mapHandle);
425
426    if (kbo->virtual == MAP_FAILED) {
427	ret = -errno;
428	goto out_err1;
429    }
430
431    return kbo;
432
433  out_err1:
434    {
435	struct ttm_pl_reference_req arg = {.handle = kbo->kBuf.handle };
436
437	(void)drmCommandWrite(slabPool->pool.fd,
438			      slabPool->devOffset + TTM_PL_UNREF,
439			      &arg, sizeof(arg));
440    }
441  out_err0:
442    free(kbo);
443    return NULL;
444}
445
446static int
447wsbmAllocSlab(struct _WsbmSlabSizeHeader *header)
448{
449    struct _WsbmSlab *slab;
450    struct _WsbmSlabBuffer *sBuf;
451    uint32_t numBuffers;
452    uint32_t ret;
453    uint32_t i;
454
455    slab = calloc(1, sizeof(*slab));
456    if (!slab)
457	return -ENOMEM;
458
459    slab->kbo = wsbmAllocKernelBO(header);
460    if (!slab->kbo) {
461	ret = -ENOMEM;
462	goto out_err0;
463    }
464
465    numBuffers = slab->kbo->actualSize / header->bufSize;
466
467    slab->buffers = calloc(numBuffers, sizeof(*slab->buffers));
468    if (!slab->buffers) {
469	ret = -ENOMEM;
470	goto out_err1;
471    }
472
473    WSBMINITLISTHEAD(&slab->head);
474    WSBMINITLISTHEAD(&slab->freeBuffers);
475    slab->numBuffers = numBuffers;
476    slab->numFree = 0;
477    slab->header = header;
478
479    sBuf = slab->buffers;
480    for (i = 0; i < numBuffers; ++i) {
481	ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool);
482	if (ret)
483	    goto out_err2;
484	sBuf->parent = slab;
485	sBuf->start = i * header->bufSize;
486	sBuf->virtual = (void *)((uint8_t *) slab->kbo->virtual +
487				 sBuf->start);
488	wsbmAtomicSet(&sBuf->writers, 0);
489	sBuf->isSlabBuffer = 1;
490	WSBM_COND_INIT(&sBuf->event);
491	WSBMLISTADDTAIL(&sBuf->head, &slab->freeBuffers);
492	slab->numFree++;
493	sBuf++;
494    }
495
496    WSBMLISTADDTAIL(&slab->head, &header->slabs);
497
498    return 0;
499
500  out_err2:
501    sBuf = slab->buffers;
502    for (i = 0; i < numBuffers; ++i) {
503	if (sBuf->parent == slab) {
504	    WSBM_COND_FREE(&sBuf->event);
505	    wsbmBufStorageTakedown(&sBuf->storage);
506	}
507	sBuf++;
508    }
509    free(slab->buffers);
510  out_err1:
511    wsbmSetKernelBOFree(header->slabPool->cache, slab->kbo);
512  out_err0:
513    free(slab);
514    return ret;
515}
516
517/*
518 * Delete a buffer from the slab header delayed list and put
519 * it on the slab free list.
520 */
521
522static void
523wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer *buf)
524{
525    struct _WsbmSlab *slab = buf->parent;
526    struct _WsbmSlabSizeHeader *header = slab->header;
527    struct _WsbmListHead *list = &buf->head;
528
529    WSBMLISTDEL(list);
530    WSBMLISTADDTAIL(list, &slab->freeBuffers);
531    slab->numFree++;
532
533    if (slab->head.next == &slab->head)
534	WSBMLISTADDTAIL(&slab->head, &header->slabs);
535
536    if (slab->numFree == slab->numBuffers) {
537	list = &slab->head;
538	WSBMLISTDEL(list);
539	WSBMLISTADDTAIL(list, &header->freeSlabs);
540    }
541
542    if (header->slabs.next == &header->slabs ||
543	slab->numFree != slab->numBuffers) {
544
545	struct _WsbmListHead *next;
546	struct _WsbmSlabCache *cache = header->slabPool->cache;
547
548	WSBMLISTFOREACHSAFE(list, next, &header->freeSlabs) {
549	    uint32_t i;
550	    struct _WsbmSlabBuffer *sBuf;
551
552	    slab = WSBMLISTENTRY(list, struct _WsbmSlab, head);
553
554	    WSBMLISTDELINIT(list);
555
556	    sBuf = slab->buffers;
557	    for (i = 0; i < slab->numBuffers; ++i) {
558		if (sBuf->parent == slab) {
559		    WSBM_COND_FREE(&sBuf->event);
560		    wsbmBufStorageTakedown(&sBuf->storage);
561		}
562		sBuf++;
563	    }
564	    wsbmSetKernelBOFree(cache, slab->kbo);
565	    free(slab->buffers);
566	    free(slab);
567	}
568    }
569}
570
571static void
572wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader *header, int wait)
573{
574  struct _WsbmListHead *list, *prev, *first, *head;
575    struct _WsbmSlabBuffer *sBuf;
576    struct _WsbmSlab *slab;
577    int firstWasSignaled = 1;
578    int signaled;
579    uint32_t i;
580    int ret;
581
582    /*
583     * Rerun the freeing test if the youngest tested buffer
584     * was signaled, since there might be more idle buffers
585     * in the delay list.
586     */
587
588    while (firstWasSignaled) {
589	firstWasSignaled = 0;
590	signaled = 0;
591	first = header->delayedBuffers.next;
592
593	/* Only examine the oldest 1/3 of delayed buffers:
594	 */
595	if (header->numDelayed > 3) {
596	    for (i = 0; i < header->numDelayed; i += 3) {
597		first = first->next;
598	    }
599	}
600
601	/*
602	 * No need to take the buffer mutex for each buffer we loop
603	 * through since we're currently the only user.
604	 */
605
606	head = first->next;
607	WSBMLISTFOREACHPREVSAFE(list, prev, head) {
608
609	    if (list == &header->delayedBuffers)
610		break;
611
612	    sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
613
614	    slab = sBuf->parent;
615
616	    if (!signaled) {
617		if (wait) {
618		    ret = wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);
619		    if (ret)
620			break;
621		    signaled = 1;
622		    wait = 0;
623		} else {
624		    signaled =
625			wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);
626#ifdef DEBUG_FENCESIGNALED
627		    fencesignaled++;
628#endif
629		}
630		if (signaled) {
631		    if (list == first)
632			firstWasSignaled = 1;
633		    wsbmFenceUnreference(&sBuf->fence);
634		    header->numDelayed--;
635		    wsbmSlabFreeBufferLocked(sBuf);
636		} else
637		    break;
638	    } else if (wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {
639		wsbmFenceUnreference(&sBuf->fence);
640		header->numDelayed--;
641		wsbmSlabFreeBufferLocked(sBuf);
642	    }
643	}
644    }
645}
646
647static struct _WsbmSlabBuffer *
648wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header)
649{
650    static struct _WsbmSlabBuffer *buf;
651    struct _WsbmSlab *slab;
652    struct _WsbmListHead *list;
653    int count = WSBM_SLABPOOL_ALLOC_RETRIES;
654
655    WSBM_MUTEX_LOCK(&header->mutex);
656    while (header->slabs.next == &header->slabs && count > 0) {
657	wsbmSlabCheckFreeLocked(header, 0);
658	if (header->slabs.next != &header->slabs)
659	    break;
660
661	WSBM_MUTEX_UNLOCK(&header->mutex);
662	if (count != WSBM_SLABPOOL_ALLOC_RETRIES)
663	    usleep(1000);
664	WSBM_MUTEX_LOCK(&header->mutex);
665	(void)wsbmAllocSlab(header);
666	count--;
667    }
668
669    list = header->slabs.next;
670    if (list == &header->slabs) {
671	WSBM_MUTEX_UNLOCK(&header->mutex);
672	return NULL;
673    }
674    slab = WSBMLISTENTRY(list, struct _WsbmSlab, head);
675    if (--slab->numFree == 0)
676	WSBMLISTDELINIT(list);
677
678    list = slab->freeBuffers.next;
679    WSBMLISTDELINIT(list);
680
681    WSBM_MUTEX_UNLOCK(&header->mutex);
682    buf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
683
684    buf->storage.destroyContainer = NULL;
685
686#ifdef DEBUG_FENCESIGNALED
687    createbuffer++;
688#endif
689    return buf;
690}
691
692static struct _WsbmBufStorage *
693pool_create(struct _WsbmBufferPool *pool, unsigned long size,
694	    uint32_t placement, unsigned alignment)
695{
696    struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool);
697    struct _WsbmSlabSizeHeader *header;
698    struct _WsbmSlabBuffer *sBuf;
699    uint32_t i;
700    int ret;
701
702    /*
703     * FIXME: Check for compatibility.
704     */
705
706    header = slabPool->headers;
707    for (i = 0; i < slabPool->numBuckets; ++i) {
708	if (header->bufSize >= size)
709	    break;
710	header++;
711    }
712
713    if (i < slabPool->numBuckets) {
714	sBuf = wsbmSlabAllocBuffer(header);
715	return ((sBuf) ? &sBuf->storage : NULL);
716    }
717
718    /*
719     * Fall back to allocate a buffer object directly from DRM.
720     * and wrap it in a wsbmBO structure.
721     */
722
723    sBuf = calloc(1, sizeof(*sBuf));
724
725    if (!sBuf)
726	return NULL;
727
728    if (alignment) {
729	if ((alignment < slabPool->pageSize)
730	    && (slabPool->pageSize % alignment))
731	    goto out_err0;
732	if ((alignment > slabPool->pageSize)
733	    && (alignment % slabPool->pageSize))
734	    goto out_err0;
735    }
736
737    ret = wsbmBufStorageInit(&sBuf->storage, pool);
738    if (ret)
739	goto out_err0;
740
741    ret = WSBM_COND_INIT(&sBuf->event);
742    if (ret)
743	goto out_err1;
744
745    {
746	union ttm_pl_create_arg arg;
747
748	arg.req.size = size;
749	arg.req.placement = placement;
750	arg.req.page_alignment = alignment / slabPool->pageSize;
751
752	DRMRESTARTCOMMANDWRITEREAD(pool->fd,
753				   slabPool->devOffset + TTM_PL_CREATE,
754				   arg, ret);
755
756	if (ret)
757	    goto out_err2;
758
759	sBuf->kBuf.gpuOffset = arg.rep.gpu_offset;
760	sBuf->kBuf.placement = arg.rep.placement;
761	sBuf->kBuf.handle = arg.rep.handle;
762	sBuf->mapHandle = arg.rep.map_handle;
763	sBuf->requestedSize = size;
764
765	sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED,
766			     pool->fd, sBuf->mapHandle);
767
768	if (sBuf->virtual == MAP_FAILED)
769	    goto out_err3;
770    }
771
772    wsbmAtomicSet(&sBuf->writers, 0);
773    return &sBuf->storage;
774  out_err3:
775    {
776	struct ttm_pl_reference_req arg;
777
778	arg.handle = sBuf->kBuf.handle;
779	(void)drmCommandWriteRead(pool->fd,
780				  slabPool->devOffset + TTM_PL_UNREF,
781				  &arg, sizeof(arg));
782    }
783  out_err2:
784    WSBM_COND_FREE(&sBuf->event);
785  out_err1:
786    wsbmBufStorageTakedown(&sBuf->storage);
787  out_err0:
788    free(sBuf);
789    return NULL;
790}
791
792static void
793pool_destroy(struct _WsbmBufStorage **p_buf)
794{
795    struct _WsbmBufStorage *buf = *p_buf;
796    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
797    struct _WsbmSlab *slab;
798    struct _WsbmSlabSizeHeader *header;
799
800    *p_buf = NULL;
801
802    if (!sBuf->isSlabBuffer) {
803	struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf);
804	struct ttm_pl_reference_req arg;
805
806	if (sBuf->virtual != NULL) {
807	    (void)munmap(sBuf->virtual, sBuf->requestedSize);
808	    sBuf->virtual = NULL;
809	}
810
811	arg.handle = sBuf->kBuf.handle;
812	(void)drmCommandWrite(slabPool->pool.fd,
813			      slabPool->devOffset + TTM_PL_UNREF,
814			      &arg, sizeof(arg));
815
816	WSBM_COND_FREE(&sBuf->event);
817	wsbmBufStorageTakedown(&sBuf->storage);
818	free(sBuf);
819	return;
820    }
821
822    slab = sBuf->parent;
823    header = slab->header;
824
825    /*
826     * No need to take the buffer mutex below since we're the only user.
827     */
828
829    WSBM_MUTEX_LOCK(&header->mutex);
830    sBuf->unFenced = 0;
831    wsbmAtomicSet(&sBuf->writers, 0);
832    wsbmAtomicSet(&sBuf->storage.refCount, 1);
833
834    if (sBuf->fence && !wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {
835	WSBMLISTADDTAIL(&sBuf->head, &header->delayedBuffers);
836	header->numDelayed++;
837    } else {
838	if (sBuf->fence)
839	    wsbmFenceUnreference(&sBuf->fence);
840	wsbmSlabFreeBufferLocked(sBuf);
841    }
842
843    WSBM_MUTEX_UNLOCK(&header->mutex);
844}
845
846static void
847waitIdleLocked(struct _WsbmSlabBuffer *sBuf, int lazy)
848{
849    struct _WsbmBufStorage *storage = &sBuf->storage;
850
851    while (sBuf->unFenced || sBuf->fence != NULL) {
852
853	if (sBuf->unFenced)
854	    WSBM_COND_WAIT(&sBuf->event, &storage->mutex);
855
856	if (sBuf->fence != NULL) {
857	    if (!wsbmFenceSignaled(sBuf->fence, sBuf->fenceType)) {
858		struct _WsbmFenceObject *fence =
859		    wsbmFenceReference(sBuf->fence);
860
861		WSBM_MUTEX_UNLOCK(&storage->mutex);
862		(void)wsbmFenceFinish(fence, sBuf->fenceType, lazy);
863		WSBM_MUTEX_LOCK(&storage->mutex);
864		if (sBuf->fence == fence)
865		    wsbmFenceUnreference(&sBuf->fence);
866
867		wsbmFenceUnreference(&fence);
868	    } else {
869		wsbmFenceUnreference(&sBuf->fence);
870	    }
871	}
872    }
873}
874
875static int
876pool_waitIdle(struct _WsbmBufStorage *buf, int lazy)
877{
878    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
879
880    WSBM_MUTEX_LOCK(&buf->mutex);
881    waitIdleLocked(sBuf, lazy);
882    WSBM_MUTEX_UNLOCK(&buf->mutex);
883
884    return 0;
885}
886
887static int
888pool_map(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)), void **virtual)
889{
890    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
891
892    *virtual = sBuf->virtual;
893
894    return 0;
895}
896
897static void
898pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)))
899{
900    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
901
902    if (wsbmAtomicDecZero(&sBuf->writers))
903	WSBM_COND_BROADCAST(&sBuf->event);
904}
905
906static int
907pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode)
908{
909    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
910    int ret = 0;
911
912    WSBM_MUTEX_LOCK(&buf->mutex);
913    if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) {
914	int signaled;
915
916	if (sBuf->unFenced) {
917	    ret = -EBUSY;
918	    goto out_unlock;
919	}
920
921	if (sBuf->isSlabBuffer)
922	    signaled = (sBuf->fence == NULL) ||
923		wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType);
924	else
925	    signaled = (sBuf->fence == NULL) ||
926		wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);
927
928	ret = 0;
929	if (signaled) {
930	    wsbmFenceUnreference(&sBuf->fence);
931	    wsbmAtomicInc(&sBuf->writers);
932	} else
933	    ret = -EBUSY;
934	goto out_unlock;
935    }
936    waitIdleLocked(sBuf, 0);
937    wsbmAtomicInc(&sBuf->writers);
938  out_unlock:
939    WSBM_MUTEX_UNLOCK(&buf->mutex);
940    return ret;
941}
942
943static void
944pool_unmap(struct _WsbmBufStorage *buf __attribute__ ((unused)))
945{
946    ;
947}
948
949static unsigned long
950pool_poolOffset(struct _WsbmBufStorage *buf)
951{
952    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
953
954    return sBuf->start;
955}
956
957static unsigned long
958pool_size(struct _WsbmBufStorage *buf)
959{
960    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
961
962    if (!sBuf->isSlabBuffer)
963	return sBuf->requestedSize;
964
965    return sBuf->parent->header->bufSize;
966}
967
968static struct _WsbmKernelBuf *
969pool_kernel(struct _WsbmBufStorage *buf)
970{
971    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
972
973    return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf;
974}
975
976static unsigned long
977pool_offset(struct _WsbmBufStorage *buf)
978{
979    return pool_kernel(buf)->gpuOffset + pool_poolOffset(buf);
980}
981
982static void
983pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence)
984{
985    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
986    struct _WsbmKernelBuf *kBuf;
987
988    WSBM_MUTEX_LOCK(&buf->mutex);
989    if (sBuf->fence)
990	wsbmFenceUnreference(&sBuf->fence);
991
992    kBuf = pool_kernel(buf);
993    sBuf->fenceType = kBuf->fence_type_mask;
994    if (!wsbmFenceSignaledCached(fence, sBuf->fenceType))
995	sBuf->fence = wsbmFenceReference(fence);
996
997    sBuf->unFenced = 0;
998    WSBM_COND_BROADCAST(&sBuf->event);
999    WSBM_MUTEX_UNLOCK(&buf->mutex);
1000}
1001
1002static int
1003pool_validate(struct _WsbmBufStorage *buf,
1004        uint64_t set_flags __attribute__ ((unused)), uint64_t clr_flags __attribute__ ((unused)))
1005{
1006    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
1007
1008    WSBM_MUTEX_LOCK(&buf->mutex);
1009    while (wsbmAtomicRead(&sBuf->writers) != 0) {
1010	WSBM_COND_WAIT(&sBuf->event, &buf->mutex);
1011    }
1012
1013    sBuf->unFenced = 1;
1014    WSBM_MUTEX_UNLOCK(&buf->mutex);
1015    return 0;
1016}
1017
1018static void
1019pool_unvalidate(struct _WsbmBufStorage *buf)
1020{
1021    struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
1022
1023    WSBM_MUTEX_LOCK(&buf->mutex);
1024    if (sBuf->unFenced) {
1025	sBuf->unFenced = 0;
1026	WSBM_COND_BROADCAST(&sBuf->event);
1027    }
1028    WSBM_MUTEX_UNLOCK(&buf->mutex);
1029}
1030
1031struct _WsbmSlabCache *
1032wsbmSlabCacheInit(uint32_t checkIntervalMsec, uint32_t slabTimeoutMsec)
1033{
1034    struct _WsbmSlabCache *tmp;
1035
1036    tmp = calloc(1, sizeof(*tmp));
1037    if (!tmp)
1038	return NULL;
1039
1040    WSBM_MUTEX_INIT(&tmp->mutex);
1041    WSBM_MUTEX_LOCK(&tmp->mutex);
1042    tmp->slabTimeout.tv_usec = slabTimeoutMsec * 1000;
1043    tmp->slabTimeout.tv_sec = tmp->slabTimeout.tv_usec / 1000000;
1044    tmp->slabTimeout.tv_usec -= tmp->slabTimeout.tv_sec * 1000000;
1045
1046    tmp->checkInterval.tv_usec = checkIntervalMsec * 1000;
1047    tmp->checkInterval.tv_sec = tmp->checkInterval.tv_usec / 1000000;
1048    tmp->checkInterval.tv_usec -= tmp->checkInterval.tv_sec * 1000000;
1049
1050    gettimeofday(&tmp->nextCheck, NULL);
1051    wsbmTimeAdd(&tmp->nextCheck, &tmp->checkInterval);
1052    WSBMINITLISTHEAD(&tmp->timeoutList);
1053    WSBMINITLISTHEAD(&tmp->unCached);
1054    WSBMINITLISTHEAD(&tmp->cached);
1055    WSBM_MUTEX_UNLOCK(&tmp->mutex);
1056
1057    return tmp;
1058}
1059
1060void
1061wsbmSlabCacheFinish(struct _WsbmSlabCache *cache)
1062{
1063    struct timeval time;
1064
1065    time = cache->nextCheck;
1066    WSBM_MUTEX_LOCK(&cache->mutex);
1067    wsbmTimeAdd(&time, &cache->checkInterval);
1068    wsbmFreeTimeoutKBOsLocked(cache, &time);
1069    WSBM_MUTEX_UNLOCK(&cache->mutex);
1070
1071    assert(cache->timeoutList.next == &cache->timeoutList);
1072    assert(cache->unCached.next == &cache->unCached);
1073    assert(cache->cached.next == &cache->cached);
1074
1075    WSBM_MUTEX_FREE(&cache->mutex);
1076    free(cache);
1077}
1078
1079static void
1080wsbmInitSizeHeader(struct _WsbmSlabPool *slabPool, uint32_t size,
1081		   struct _WsbmSlabSizeHeader *header)
1082{
1083    WSBM_MUTEX_INIT(&header->mutex);
1084    WSBM_MUTEX_LOCK(&header->mutex);
1085
1086    WSBMINITLISTHEAD(&header->slabs);
1087    WSBMINITLISTHEAD(&header->freeSlabs);
1088    WSBMINITLISTHEAD(&header->delayedBuffers);
1089
1090    header->numDelayed = 0;
1091    header->slabPool = slabPool;
1092    header->bufSize = size;
1093
1094    WSBM_MUTEX_UNLOCK(&header->mutex);
1095}
1096
1097static void
1098wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader *header)
1099{
1100    struct _WsbmListHead *list, *next;
1101    struct _WsbmSlabBuffer *sBuf;
1102
1103    WSBM_MUTEX_LOCK(&header->mutex);
1104    WSBMLISTFOREACHSAFE(list, next, &header->delayedBuffers) {
1105	sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
1106
1107	if (sBuf->fence) {
1108	    (void)wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);
1109	    wsbmFenceUnreference(&sBuf->fence);
1110	}
1111	header->numDelayed--;
1112	wsbmSlabFreeBufferLocked(sBuf);
1113    }
1114    WSBM_MUTEX_UNLOCK(&header->mutex);
1115    WSBM_MUTEX_FREE(&header->mutex);
1116}
1117
1118static void
1119pool_takedown(struct _WsbmBufferPool *pool)
1120{
1121    struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool);
1122    unsigned int i;
1123
1124    for (i = 0; i < slabPool->numBuckets; ++i) {
1125	wsbmFinishSizeHeader(&slabPool->headers[i]);
1126    }
1127
1128    free(slabPool->headers);
1129    free(slabPool->bucketSizes);
1130    free(slabPool);
1131}
1132
1133struct _WsbmBufferPool *
1134wsbmSlabPoolInit(int fd,
1135		 uint32_t devOffset,
1136		 uint32_t placement,
1137		 uint32_t validMask,
1138		 uint32_t smallestSize,
1139		 uint32_t numSizes,
1140		 uint32_t desiredNumBuffers,
1141		 uint32_t maxSlabSize,
1142		 uint32_t pageAlignment, struct _WsbmSlabCache *cache)
1143{
1144    struct _WsbmBufferPool *pool;
1145    struct _WsbmSlabPool *slabPool;
1146    uint32_t i;
1147
1148    slabPool = calloc(1, sizeof(*slabPool));
1149    if (!slabPool)
1150	return NULL;
1151
1152    pool = &slabPool->pool;
1153
1154    slabPool->bucketSizes = calloc(numSizes, sizeof(*slabPool->bucketSizes));
1155    if (!slabPool->bucketSizes)
1156	goto out_err0;
1157
1158    slabPool->headers = calloc(numSizes, sizeof(*slabPool->headers));
1159    if (!slabPool->headers)
1160	goto out_err1;
1161
1162    slabPool->devOffset = devOffset;
1163    slabPool->cache = cache;
1164    slabPool->proposedPlacement = placement;
1165    slabPool->validMask = validMask;
1166    slabPool->numBuckets = numSizes;
1167    slabPool->pageSize = getpagesize();
1168    slabPool->pageAlignment = pageAlignment;
1169    slabPool->maxSlabSize = maxSlabSize;
1170    slabPool->desiredNumBuffers = desiredNumBuffers;
1171
1172    for (i = 0; i < slabPool->numBuckets; ++i) {
1173	slabPool->bucketSizes[i] = (smallestSize << i);
1174	wsbmInitSizeHeader(slabPool, slabPool->bucketSizes[i],
1175			   &slabPool->headers[i]);
1176    }
1177
1178    pool->fd = fd;
1179    pool->map = &pool_map;
1180    pool->unmap = &pool_unmap;
1181    pool->destroy = &pool_destroy;
1182    pool->offset = &pool_offset;
1183    pool->poolOffset = &pool_poolOffset;
1184    pool->size = &pool_size;
1185    pool->create = &pool_create;
1186    pool->fence = &pool_fence;
1187    pool->kernel = &pool_kernel;
1188    pool->validate = &pool_validate;
1189    pool->unvalidate = &pool_unvalidate;
1190    pool->waitIdle = &pool_waitIdle;
1191    pool->takeDown = &pool_takedown;
1192    pool->releasefromcpu = &pool_releaseFromCpu;
1193    pool->syncforcpu = &pool_syncForCpu;
1194
1195    return pool;
1196
1197  out_err1:
1198    free(slabPool->bucketSizes);
1199  out_err0:
1200    free(slabPool);
1201
1202    return NULL;
1203}
1204