1/**************************************************************************
2 *
3 * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA
4 * All Rights Reserved.
5 * Copyright 2009 VMware, Inc., Palo Alto, CA., USA
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
23 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
24 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
25 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
26 * USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
28 **************************************************************************/
29/*
30 * Authors: Thomas Hellstr�m <thomas-at-tungstengraphics-dot-com>
31 */
32#ifdef HAVE_CONFIG_H
33#include "config.h"
34#endif
35
36#include <xf86drm.h>
37#include <stdlib.h>
38#include <unistd.h>
39#include <errno.h>
40#include <sys/mman.h>
41#include <drm/psb_ttm_placement_user.h>
42#include "wsbm_pool.h"
43#include "assert.h"
44#include "wsbm_priv.h"
45#include "wsbm_manager.h"
46
47#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret)			\
48	do {								\
49		(_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \
50	} while ((_ret) == -EAGAIN || (_ret) == -ERESTART);		\
51
52#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret)		\
53	do {								\
54		(_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \
55	} while ((_ret) == -EAGAIN || (_ret) == -ERESTART);		\
56
57/*
58 * Buffer pool implementation using DRM buffer objects as wsbm buffer objects.
59 */
60
61struct _TTMBuffer
62{
63    struct _WsbmBufStorage buf;
64    struct _WsbmCond event;
65
66    /*
67     * Remains constant after creation.
68     */
69
70    uint64_t requestedSize;
71    uint64_t mapHandle;
72    uint64_t realSize;
73
74    /*
75     * Protected by the kernel lock.
76     */
77
78    struct _WsbmKernelBuf kBuf;
79
80    /*
81     * Protected by the mutex.
82     */
83
84    void *virtual;
85    int syncInProgress;
86    unsigned readers;
87    unsigned writers;
88};
89
90struct _TTMPool
91{
92    struct _WsbmBufferPool pool;
93    unsigned int pageSize;
94    unsigned int devOffset;
95};
96
97static inline struct _TTMPool *
98ttmGetPool(struct _TTMBuffer *dBuf)
99{
100    return containerOf(dBuf->buf.pool, struct _TTMPool, pool);
101}
102
103static inline struct _TTMBuffer *
104ttmBuffer(struct _WsbmBufStorage *buf)
105{
106    return containerOf(buf, struct _TTMBuffer, buf);
107}
108
109static struct _WsbmBufStorage *
110pool_create(struct _WsbmBufferPool *pool,
111	    unsigned long size, uint32_t placement, unsigned alignment)
112{
113    struct _TTMBuffer *dBuf = (struct _TTMBuffer *)
114	calloc(1, sizeof(*dBuf));
115    struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool);
116    int ret;
117    unsigned pageSize = ttmPool->pageSize;
118    union ttm_pl_create_arg arg;
119
120    if (!dBuf)
121	return NULL;
122
123    if ((alignment > pageSize) && (alignment % pageSize))
124	goto out_err0;
125
126    ret = wsbmBufStorageInit(&dBuf->buf, pool);
127    if (ret)
128	goto out_err0;
129
130    ret = WSBM_COND_INIT(&dBuf->event);
131    if (ret)
132	goto out_err1;
133
134    arg.req.size = size;
135    arg.req.placement = placement;
136    arg.req.page_alignment = alignment / pageSize;
137
138    DRMRESTARTCOMMANDWRITEREAD(pool->fd, ttmPool->devOffset + TTM_PL_CREATE,
139			       arg, ret);
140
141    if (ret)
142	goto out_err2;
143
144    dBuf->requestedSize = size;
145    dBuf->kBuf.gpuOffset = arg.rep.gpu_offset;
146    dBuf->mapHandle = arg.rep.map_handle;
147    dBuf->realSize = arg.rep.bo_size;
148    dBuf->kBuf.placement = arg.rep.placement;
149    dBuf->kBuf.handle = arg.rep.handle;
150
151    return &dBuf->buf;
152
153  out_err2:
154    WSBM_COND_FREE(&dBuf->event);
155  out_err1:
156    wsbmBufStorageTakedown(&dBuf->buf);
157  out_err0:
158    free(dBuf);
159    return NULL;
160}
161
162static struct _WsbmBufStorage *
163pool_reference(struct _WsbmBufferPool *pool, unsigned handle)
164{
165    struct _TTMBuffer *dBuf = (struct _TTMBuffer *)calloc(1, sizeof(*dBuf));
166    struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool);
167    union ttm_pl_reference_arg arg;
168    int ret;
169
170    if (!dBuf)
171	return NULL;
172
173    ret = wsbmBufStorageInit(&dBuf->buf, pool);
174    if (ret)
175	goto out_err0;
176
177    ret = WSBM_COND_INIT(&dBuf->event);
178    if (ret)
179	goto out_err1;
180
181    arg.req.handle = handle;
182    ret = drmCommandWriteRead(pool->fd, ttmPool->devOffset + TTM_PL_REFERENCE,
183			      &arg, sizeof(arg));
184
185    if (ret)
186	goto out_err2;
187
188    dBuf->requestedSize = arg.rep.bo_size;
189    dBuf->kBuf.gpuOffset = arg.rep.gpu_offset;
190    dBuf->mapHandle = arg.rep.map_handle;
191    dBuf->realSize = arg.rep.bo_size;
192    dBuf->kBuf.placement = arg.rep.placement;
193    dBuf->kBuf.handle = arg.rep.handle;
194    dBuf->kBuf.fence_type_mask = arg.rep.sync_object_arg;
195
196    return &dBuf->buf;
197
198  out_err2:
199    WSBM_COND_FREE(&dBuf->event);
200  out_err1:
201    wsbmBufStorageTakedown(&dBuf->buf);
202  out_err0:
203    free(dBuf);
204    return NULL;
205}
206
207static void
208pool_destroy(struct _WsbmBufStorage **buf)
209{
210    struct _TTMBuffer *dBuf = ttmBuffer(*buf);
211    struct _TTMPool *ttmPool = ttmGetPool(dBuf);
212    struct ttm_pl_reference_req arg;
213
214    *buf = NULL;
215    if (dBuf->virtual != NULL) {
216	(void)munmap(dBuf->virtual, dBuf->requestedSize);
217	dBuf->virtual = NULL;
218    }
219    arg.handle = dBuf->kBuf.handle;
220    (void)drmCommandWrite(dBuf->buf.pool->fd,
221			  ttmPool->devOffset + TTM_PL_UNREF,
222			  &arg, sizeof(arg));
223
224    WSBM_COND_FREE(&dBuf->event);
225    wsbmBufStorageTakedown(&dBuf->buf);
226    free(dBuf);
227}
228
229static int
230syncforcpu_locked(struct _WsbmBufStorage *buf, unsigned mode)
231{
232    uint32_t kmode = 0;
233    struct _TTMBuffer *dBuf = ttmBuffer(buf);
234    struct _TTMPool *ttmPool = ttmGetPool(dBuf);
235    unsigned int readers;
236    unsigned int writers;
237    int ret = 0;
238
239    while (dBuf->syncInProgress)
240	WSBM_COND_WAIT(&dBuf->event, &buf->mutex);
241
242    readers = dBuf->readers;
243    writers = dBuf->writers;
244
245    if ((mode & WSBM_SYNCCPU_READ) && (++dBuf->readers == 1))
246	kmode |= TTM_PL_SYNCCPU_MODE_READ;
247
248    if ((mode & WSBM_SYNCCPU_WRITE) && (++dBuf->writers == 1))
249	kmode |= TTM_PL_SYNCCPU_MODE_WRITE;
250
251    if (kmode) {
252	struct ttm_pl_synccpu_arg arg;
253
254	if (mode & WSBM_SYNCCPU_DONT_BLOCK)
255	    kmode |= TTM_PL_SYNCCPU_MODE_NO_BLOCK;
256
257	dBuf->syncInProgress = 1;
258
259	/*
260	 * This might be a lengthy wait, so
261	 * release the mutex.
262	 */
263
264	WSBM_MUTEX_UNLOCK(&buf->mutex);
265
266	arg.handle = dBuf->kBuf.handle;
267	arg.access_mode = kmode;
268	arg.op = TTM_PL_SYNCCPU_OP_GRAB;
269
270	DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd,
271			       ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret);
272
273	WSBM_MUTEX_LOCK(&buf->mutex);
274	dBuf->syncInProgress = 0;
275	WSBM_COND_BROADCAST(&dBuf->event);
276
277	if (ret) {
278	    dBuf->readers = readers;
279	    dBuf->writers = writers;
280	}
281    }
282
283    return ret;
284}
285
286static int
287releasefromcpu_locked(struct _WsbmBufStorage *buf, unsigned mode)
288{
289    uint32_t kmode = 0;
290    struct _TTMBuffer *dBuf = ttmBuffer(buf);
291    struct _TTMPool *ttmPool = ttmGetPool(dBuf);
292    int ret = 0;
293
294    while (dBuf->syncInProgress)
295	WSBM_COND_WAIT(&dBuf->event, &buf->mutex);
296
297    if ((mode & WSBM_SYNCCPU_READ) && (--dBuf->readers == 0))
298	kmode |= TTM_PL_SYNCCPU_MODE_READ;
299
300    if ((mode & WSBM_SYNCCPU_WRITE) && (--dBuf->writers == 0))
301	kmode |= TTM_PL_SYNCCPU_MODE_WRITE;
302
303    if (kmode) {
304	struct ttm_pl_synccpu_arg arg;
305
306	arg.handle = dBuf->kBuf.handle;
307	arg.access_mode = kmode;
308	arg.op = TTM_PL_SYNCCPU_OP_RELEASE;
309
310	DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd,
311			       ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret);
312
313    }
314
315    return ret;
316}
317
318static int
319pool_syncforcpu(struct _WsbmBufStorage *buf, unsigned mode)
320{
321    int ret;
322
323    WSBM_MUTEX_LOCK(&buf->mutex);
324    ret = syncforcpu_locked(buf, mode);
325    WSBM_MUTEX_UNLOCK(&buf->mutex);
326    return ret;
327}
328
329static void
330pool_releasefromcpu(struct _WsbmBufStorage *buf, unsigned mode)
331{
332    WSBM_MUTEX_LOCK(&buf->mutex);
333    (void)releasefromcpu_locked(buf, mode);
334    WSBM_MUTEX_UNLOCK(&buf->mutex);
335}
336
337#ifdef ANDROID
338
339/* No header but syscall provided by bionic */
340void*  __mmap2(void*, size_t, int, int, int, size_t);
341#define MMAP2_SHIFT 12 // 2**12 == 4096
342
343static void* _temp_mmap(void *addr, size_t size, int prot, int flags, int fd, long long offset)
344{
345    return __mmap2(addr, size, prot, flags, fd, (unsigned long)(offset >> MMAP2_SHIFT));
346}
347
348#endif
349
350static int
351pool_map(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)), void **virtual)
352{
353    struct _TTMBuffer *dBuf = ttmBuffer(buf);
354    void *virt;
355    int ret = 0;
356
357    WSBM_MUTEX_LOCK(&buf->mutex);
358
359    /*
360     * mmaps are expensive, so we only really unmap if
361     * we destroy the buffer.
362     */
363
364    if (dBuf->virtual == NULL) {
365#if defined(__LP64__) || defined(_LP64) || defined(__LP64)
366	virt = mmap(0, dBuf->requestedSize,
367		    PROT_READ | PROT_WRITE, MAP_SHARED,
368		    buf->pool->fd, dBuf->mapHandle);
369#else
370	virt = _temp_mmap(0, dBuf->requestedSize,
371		    PROT_READ | PROT_WRITE, MAP_SHARED,
372		    buf->pool->fd, dBuf->mapHandle);
373#endif
374	if (virt == MAP_FAILED) {
375	    ret = -errno;
376	    goto out_unlock;
377	}
378	dBuf->virtual = virt;
379    }
380
381    *virtual = dBuf->virtual;
382  out_unlock:
383
384    WSBM_MUTEX_UNLOCK(&buf->mutex);
385
386    return ret;
387}
388
389static void
390pool_unmap(struct _WsbmBufStorage *buf __attribute__ ((unused)))
391{
392    ;
393}
394
395static unsigned long
396pool_offset(struct _WsbmBufStorage *buf)
397{
398    struct _TTMBuffer *dBuf = ttmBuffer(buf);
399
400    return dBuf->kBuf.gpuOffset;
401}
402
403static unsigned long
404pool_poolOffset(struct _WsbmBufStorage *buf __attribute__ ((unused)))
405{
406    return 0;
407}
408
409static uint32_t
410pool_placement(struct _WsbmBufStorage *buf)
411{
412    struct _TTMBuffer *dBuf = ttmBuffer(buf);
413
414    return dBuf->kBuf.placement;
415}
416
417static unsigned long
418pool_size(struct _WsbmBufStorage *buf)
419{
420    struct _TTMBuffer *dBuf = ttmBuffer(buf);
421
422    return dBuf->realSize;
423}
424
425static void
426pool_fence(struct _WsbmBufStorage *buf __attribute__ ((unused)),
427        struct _WsbmFenceObject *fence __attribute__ ((unused)))
428{
429    /*
430     * Noop. The kernel handles all fencing.
431     */
432}
433
434static int
435pool_waitIdle(struct _WsbmBufStorage *buf, int lazy)
436{
437    struct _TTMBuffer *dBuf = ttmBuffer(buf);
438    struct _TTMPool *ttmPool = ttmGetPool(dBuf);
439    struct ttm_pl_waitidle_arg req;
440    struct _WsbmBufferPool *pool = buf->pool;
441    int ret;
442
443    req.handle = dBuf->kBuf.handle;
444    req.mode = (lazy) ? TTM_PL_WAITIDLE_MODE_LAZY : 0;
445
446    DRMRESTARTCOMMANDWRITE(pool->fd, ttmPool->devOffset + TTM_PL_WAITIDLE,
447			   req, ret);
448
449    return ret;
450}
451
452static void
453pool_takedown(struct _WsbmBufferPool *pool)
454{
455    struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool);
456
457    free(ttmPool);
458}
459
460static int
461pool_setStatus(struct _WsbmBufStorage *buf, uint32_t set_placement,
462	       uint32_t clr_placement)
463{
464    struct _TTMBuffer *dBuf = ttmBuffer(buf);
465    struct _TTMPool *ttmPool = ttmGetPool(dBuf);
466    union ttm_pl_setstatus_arg arg;
467    struct ttm_pl_setstatus_req *req = &arg.req;
468    struct ttm_pl_rep *rep = &arg.rep;
469    struct _WsbmBufferPool *pool = buf->pool;
470    int ret;
471
472    req->handle = dBuf->kBuf.handle;
473    req->set_placement = set_placement;
474    req->clr_placement = clr_placement;
475
476    DRMRESTARTCOMMANDWRITEREAD(pool->fd,
477			       ttmPool->devOffset + TTM_PL_SETSTATUS,
478			       arg, ret);
479
480    if (!ret) {
481	dBuf->kBuf.gpuOffset = rep->gpu_offset;
482	dBuf->kBuf.placement = rep->placement;
483    }
484
485    return ret;
486}
487
488static struct _WsbmKernelBuf *
489pool_kernel(struct _WsbmBufStorage *buf)
490{
491    return (void *)&ttmBuffer(buf)->kBuf;
492}
493
494struct _WsbmBufferPool *
495wsbmTTMPoolInit(int fd, unsigned int devOffset)
496{
497    struct _TTMPool *ttmPool;
498    struct _WsbmBufferPool *pool;
499
500    ttmPool = (struct _TTMPool *)calloc(1, sizeof(*ttmPool));
501
502    if (!ttmPool)
503	return NULL;
504
505    ttmPool->pageSize = getpagesize();
506    ttmPool->devOffset = devOffset;
507    pool = &ttmPool->pool;
508
509    pool->fd = fd;
510    pool->map = &pool_map;
511    pool->unmap = &pool_unmap;
512    pool->syncforcpu = &pool_syncforcpu;
513    pool->releasefromcpu = &pool_releasefromcpu;
514    pool->destroy = &pool_destroy;
515    pool->offset = &pool_offset;
516    pool->poolOffset = &pool_poolOffset;
517    pool->placement = &pool_placement;
518    pool->size = &pool_size;
519    pool->create = &pool_create;
520    pool->fence = &pool_fence;
521    pool->kernel = &pool_kernel;
522    pool->validate = NULL;
523    pool->unvalidate = NULL;
524    pool->waitIdle = &pool_waitIdle;
525    pool->takeDown = &pool_takedown;
526    pool->createByReference = &pool_reference;
527    pool->setStatus = &pool_setStatus;
528    return pool;
529}
530
531struct _WsbmBufStorage *
532ttm_pool_ub_create(struct _WsbmBufferPool *pool, unsigned long size, uint32_t placement, unsigned alignment, const unsigned long *user_ptr, int fd)
533{
534    struct _TTMBuffer *dBuf = (struct _TTMBuffer *)
535	    calloc(1, sizeof(*dBuf));
536    struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool);
537    int ret;
538    unsigned pageSize = ttmPool->pageSize;
539    union ttm_pl_create_ub_arg arg;
540
541    if (!dBuf)
542	    return NULL;
543
544    if ((alignment > pageSize) && (alignment % pageSize))
545	    goto out_err0;
546
547    ret = wsbmBufStorageInit(&dBuf->buf, pool);
548    if (ret)
549	    goto out_err0;
550
551    ret = WSBM_COND_INIT(&dBuf->event);
552    if (ret)
553	    goto out_err1;
554
555    arg.req.size = size;
556    arg.req.placement = placement;
557    arg.req.page_alignment = alignment / pageSize;
558    arg.req.user_address = (unsigned long)user_ptr;
559    arg.req.fd = fd;
560
561    DRMRESTARTCOMMANDWRITEREAD(pool->fd, ttmPool->devOffset + TTM_PL_CREATE_UB,
562			       arg, ret);
563    if (ret)
564        goto out_err2;
565
566    dBuf->requestedSize = size;
567    dBuf->kBuf.gpuOffset = arg.rep.gpu_offset;
568    dBuf->mapHandle = arg.rep.map_handle;
569    dBuf->realSize = arg.rep.bo_size;
570    dBuf->kBuf.placement = arg.rep.placement;
571    dBuf->kBuf.handle = arg.rep.handle;
572
573    return &dBuf->buf;
574
575  out_err2:
576    WSBM_COND_FREE(&dBuf->event);
577  out_err1:
578    wsbmBufStorageTakedown(&dBuf->buf);
579  out_err0:
580    free(dBuf);
581    return NULL;
582}
583
584