radeon_drm_bo.c revision 032b162ce88ef6ec8ad981fff709eb177d794589 
1#define _FILE_OFFSET_BITS 64
2#include "radeon_drm_cs.h"
3
4#include "util/u_hash_table.h"
5#include "util/u_memory.h"
6#include "util/u_simple_list.h"
7#include "os/os_thread.h"
8
9#include "state_tracker/drm_driver.h"
10
11#include <sys/ioctl.h>
12#include <sys/mman.h>
13#include <xf86drm.h>
14#include <errno.h>
15
16#define RADEON_BO_FLAGS_MACRO_TILE  1
17#define RADEON_BO_FLAGS_MICRO_TILE  2
18#define RADEON_BO_FLAGS_MICRO_TILE_SQUARE 0x20
19
20extern const struct pb_vtbl radeon_bo_vtbl;
21
22
23static INLINE struct radeon_bo *radeon_bo(struct pb_buffer *bo)
24{
25    assert(bo->vtbl == &radeon_bo_vtbl);
26    return (struct radeon_bo *)bo;
27}
28
29struct radeon_bomgr {
30    /* Base class. */
31    struct pb_manager base;
32
33    /* Winsys. */
34    struct radeon_drm_winsys *rws;
35
36    /* List of buffer handles and its mutex. */
37    struct util_hash_table *bo_handles;
38    pipe_mutex bo_handles_mutex;
39};
40
41static INLINE struct radeon_bomgr *radeon_bomgr(struct pb_manager *mgr)
42{
43    return (struct radeon_bomgr *)mgr;
44}
45
46static struct radeon_bo *get_radeon_bo(struct pb_buffer *_buf)
47{
48    struct radeon_bo *bo = NULL;
49
50    if (_buf->vtbl == &radeon_bo_vtbl) {
51        bo = radeon_bo(_buf);
52    } else {
53	struct pb_buffer *base_buf;
54	pb_size offset;
55	pb_get_base_buffer(_buf, &base_buf, &offset);
56
57        if (base_buf->vtbl == &radeon_bo_vtbl)
58            bo = radeon_bo(base_buf);
59    }
60
61    return bo;
62}
63
64void radeon_bo_unref(struct radeon_bo *bo)
65{
66    struct drm_gem_close args = {};
67
68    if (!p_atomic_dec_zero(&bo->cref))
69        return;
70
71    if (bo->name) {
72        pipe_mutex_lock(bo->mgr->bo_handles_mutex);
73        util_hash_table_remove(bo->mgr->bo_handles,
74			       (void*)(uintptr_t)bo->name);
75        pipe_mutex_unlock(bo->mgr->bo_handles_mutex);
76    }
77
78    if (bo->ptr)
79        munmap(bo->ptr, bo->size);
80
81    /* Close object. */
82    args.handle = bo->handle;
83    drmIoctl(bo->mgr->rws->fd, DRM_IOCTL_GEM_CLOSE, &args);
84    FREE(bo);
85}
86
87static void radeon_bo_wait(struct r300_winsys_bo *_buf)
88{
89    struct radeon_bo *bo = get_radeon_bo(pb_buffer(_buf));
90    struct drm_radeon_gem_wait_idle args = {};
91
92    args.handle = bo->handle;
93    while (drmCommandWriteRead(bo->mgr->rws->fd, DRM_RADEON_GEM_WAIT_IDLE,
94                               &args, sizeof(args)) == -EBUSY);
95}
96
97static boolean radeon_bo_is_busy(struct r300_winsys_bo *_buf)
98{
99    struct radeon_bo *bo = get_radeon_bo(pb_buffer(_buf));
100    struct drm_radeon_gem_busy args = {};
101
102    args.handle = bo->handle;
103    return drmCommandWriteRead(bo->mgr->rws->fd, DRM_RADEON_GEM_BUSY,
104                               &args, sizeof(args)) != 0;
105}
106
107static void radeon_bo_destroy(struct pb_buffer *_buf)
108{
109    struct radeon_bo *bo = radeon_bo(_buf);
110
111    radeon_bo_unref(bo);
112}
113
114static unsigned get_pb_usage_from_transfer_flags(enum pipe_transfer_usage usage)
115{
116    unsigned res = 0;
117
118    if (usage & PIPE_TRANSFER_DONTBLOCK)
119        res |= PB_USAGE_DONTBLOCK;
120
121    if (usage & PIPE_TRANSFER_UNSYNCHRONIZED)
122        res |= PB_USAGE_UNSYNCHRONIZED;
123
124    return res;
125}
126
127static void *radeon_bo_map_internal(struct pb_buffer *_buf,
128                                    unsigned flags, void *flush_ctx)
129{
130    struct radeon_bo *bo = radeon_bo(_buf);
131    struct radeon_drm_cs *cs = flush_ctx;
132    struct drm_radeon_gem_mmap args = {};
133
134    if (flags & PB_USAGE_DONTBLOCK) {
135        /* Note how we use radeon_bo_is_referenced_by_cs here. There are
136         * basically two places this map function can be called from:
137         * - pb_map
138         * - create_buffer (in the buffer reuse case)
139         *
140         * Since pb managers are per-winsys managers, not per-context managers,
141         * and we shouldn't reuse buffers if they are in-use in any context,
142         * we simply ask: is this buffer referenced by *any* CS?
143         *
144         * The problem with buffer_create is that it comes from pipe_screen,
145         * so we have no CS to look at, though luckily the following code
146         * is sufficient to tell whether the buffer is in use. */
147        if (_buf->base.usage & RADEON_PB_USAGE_CACHE) {
148            if (radeon_bo_is_referenced_by_any_cs(bo))
149		return NULL;
150	}
151
152        if (cs && radeon_bo_is_referenced_by_cs(cs, bo)) {
153            cs->flush_cs(cs->flush_data);
154            return NULL; /* It's very unlikely that the buffer is not busy. */
155        }
156
157        if (radeon_bo_is_busy((struct r300_winsys_bo*)bo)) {
158            return NULL;
159        }
160    }
161
162    /* If it's not unsynchronized bo_map, flush CS if needed and then wait. */
163    if (!(flags & PB_USAGE_UNSYNCHRONIZED)) {
164        if (cs && radeon_bo_is_referenced_by_cs(cs, bo)) {
165            cs->flush_cs(cs->flush_data);
166        }
167
168        radeon_bo_wait((struct r300_winsys_bo*)bo);
169    }
170
171    /* Map buffer if it's not already mapped. */
172    /* XXX We may get a race in bo->ptr. */
173    if (!bo->ptr) {
174        void *ptr;
175
176        args.handle = bo->handle;
177        args.offset = 0;
178        args.size = (uint64_t)bo->size;
179        if (drmCommandWriteRead(bo->mgr->rws->fd,
180                                DRM_RADEON_GEM_MMAP,
181                                &args,
182                                sizeof(args))) {
183            fprintf(stderr, "radeon: gem_mmap failed: %p 0x%08X\n",
184                    bo, bo->handle);
185            return NULL;
186        }
187        ptr = mmap(0, args.size, PROT_READ|PROT_WRITE, MAP_SHARED,
188                   bo->mgr->rws->fd, args.addr_ptr);
189        if (ptr == MAP_FAILED) {
190            fprintf(stderr, "radeon: mmap failed, errno: %i\n", errno);
191            return NULL;
192        }
193        bo->ptr = ptr;
194    }
195
196    return bo->ptr;
197}
198
199static void radeon_bo_unmap_internal(struct pb_buffer *_buf)
200{
201    /* NOP */
202}
203
204static void radeon_bo_get_base_buffer(struct pb_buffer *buf,
205				      struct pb_buffer **base_buf,
206				      unsigned *offset)
207{
208    *base_buf = buf;
209    *offset = 0;
210}
211
212static enum pipe_error radeon_bo_validate(struct pb_buffer *_buf,
213					  struct pb_validate *vl,
214					  unsigned flags)
215{
216    /* Always pinned */
217    return PIPE_OK;
218}
219
220static void radeon_bo_fence(struct pb_buffer *buf,
221                            struct pipe_fence_handle *fence)
222{
223}
224
225const struct pb_vtbl radeon_bo_vtbl = {
226    radeon_bo_destroy,
227    radeon_bo_map_internal,
228    radeon_bo_unmap_internal,
229    radeon_bo_validate,
230    radeon_bo_fence,
231    radeon_bo_get_base_buffer,
232};
233
234static struct pb_buffer *radeon_bomgr_create_bo(struct pb_manager *_mgr,
235						pb_size size,
236						const struct pb_desc *desc)
237{
238    struct radeon_bomgr *mgr = radeon_bomgr(_mgr);
239    struct radeon_drm_winsys *rws = mgr->rws;
240    struct radeon_bo *bo;
241    struct drm_radeon_gem_create args = {};
242
243    args.size = size;
244    args.alignment = desc->alignment;
245    args.initial_domain =
246        (desc->usage & RADEON_PB_USAGE_DOMAIN_GTT  ?
247         RADEON_GEM_DOMAIN_GTT  : 0) |
248        (desc->usage & RADEON_PB_USAGE_DOMAIN_VRAM ?
249         RADEON_GEM_DOMAIN_VRAM : 0);
250
251    if (drmCommandWriteRead(rws->fd, DRM_RADEON_GEM_CREATE,
252                            &args, sizeof(args))) {
253        fprintf(stderr, "Failed to allocate :\n");
254        fprintf(stderr, "   size      : %d bytes\n", size);
255        fprintf(stderr, "   alignment : %d bytes\n", desc->alignment);
256        fprintf(stderr, "   domains   : %d\n", args.initial_domain);
257        return NULL;
258    }
259
260    bo = CALLOC_STRUCT(radeon_bo);
261    if (!bo)
262	return NULL;
263
264    pipe_reference_init(&bo->base.base.reference, 1);
265    bo->base.base.alignment = desc->alignment;
266    bo->base.base.usage = desc->usage;
267    bo->base.base.size = size;
268    bo->base.vtbl = &radeon_bo_vtbl;
269    bo->mgr = mgr;
270    bo->handle = args.handle;
271    bo->size = size;
272
273    radeon_bo_ref(bo);
274    return &bo->base;
275}
276
277static void radeon_bomgr_flush(struct pb_manager *mgr)
278{
279    /* NOP */
280}
281
282static void radeon_bomgr_destroy(struct pb_manager *_mgr)
283{
284    struct radeon_bomgr *mgr = radeon_bomgr(_mgr);
285    util_hash_table_destroy(mgr->bo_handles);
286    pipe_mutex_destroy(mgr->bo_handles_mutex);
287    FREE(mgr);
288}
289
290static unsigned handle_hash(void *key)
291{
292    return (unsigned)key;
293}
294
295static int handle_compare(void *key1, void *key2)
296{
297    return !((int)key1 == (int)key2);
298}
299
300struct pb_manager *radeon_bomgr_create(struct radeon_drm_winsys *rws)
301{
302    struct radeon_bomgr *mgr;
303
304    mgr = CALLOC_STRUCT(radeon_bomgr);
305    if (!mgr)
306	return NULL;
307
308    mgr->base.destroy = radeon_bomgr_destroy;
309    mgr->base.create_buffer = radeon_bomgr_create_bo;
310    mgr->base.flush = radeon_bomgr_flush;
311
312    mgr->rws = rws;
313    mgr->bo_handles = util_hash_table_create(handle_hash, handle_compare);
314    pipe_mutex_init(mgr->bo_handles_mutex);
315    return &mgr->base;
316}
317
318static void *radeon_bo_map(struct r300_winsys_bo *buf,
319                           struct r300_winsys_cs *cs,
320                           enum pipe_transfer_usage usage)
321{
322    struct pb_buffer *_buf = pb_buffer(buf);
323
324    return pb_map(_buf, get_pb_usage_from_transfer_flags(usage), cs);
325}
326
327static void radeon_bo_get_tiling(struct r300_winsys_bo *_buf,
328                                 enum r300_buffer_tiling *microtiled,
329                                 enum r300_buffer_tiling *macrotiled)
330{
331    struct radeon_bo *bo = get_radeon_bo(pb_buffer(_buf));
332    struct drm_radeon_gem_set_tiling args = {};
333
334    args.handle = bo->handle;
335
336    drmCommandWriteRead(bo->mgr->rws->fd,
337                        DRM_RADEON_GEM_GET_TILING,
338                        &args,
339                        sizeof(args));
340
341    *microtiled = R300_BUFFER_LINEAR;
342    *macrotiled = R300_BUFFER_LINEAR;
343    if (args.tiling_flags & RADEON_BO_FLAGS_MICRO_TILE)
344	*microtiled = R300_BUFFER_TILED;
345
346    if (args.tiling_flags & RADEON_BO_FLAGS_MACRO_TILE)
347	*macrotiled = R300_BUFFER_TILED;
348}
349
350static void radeon_bo_set_tiling(struct r300_winsys_bo *_buf,
351                                 enum r300_buffer_tiling microtiled,
352                                 enum r300_buffer_tiling macrotiled,
353                                 uint32_t pitch)
354{
355    struct radeon_bo *bo = get_radeon_bo(pb_buffer(_buf));
356    struct drm_radeon_gem_set_tiling args = {};
357
358    if (microtiled == R300_BUFFER_TILED)
359        args.tiling_flags |= RADEON_BO_FLAGS_MICRO_TILE;
360    else if (microtiled == R300_BUFFER_SQUARETILED)
361        args.tiling_flags |= RADEON_BO_FLAGS_MICRO_TILE_SQUARE;
362
363    if (macrotiled == R300_BUFFER_TILED)
364        args.tiling_flags |= RADEON_BO_FLAGS_MACRO_TILE;
365
366    args.handle = bo->handle;
367    args.pitch = pitch;
368
369    drmCommandWriteRead(bo->mgr->rws->fd,
370                        DRM_RADEON_GEM_SET_TILING,
371                        &args,
372                        sizeof(args));
373}
374
375static struct r300_winsys_cs_handle *radeon_drm_get_cs_handle(
376        struct r300_winsys_bo *_buf)
377{
378    /* return radeon_bo. */
379    return (struct r300_winsys_cs_handle*)
380            get_radeon_bo(pb_buffer(_buf));
381}
382
383static unsigned get_pb_usage_from_create_flags(unsigned bind, unsigned usage,
384                                               enum r300_buffer_domain domain)
385{
386    unsigned res = 0;
387
388    if (bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
389        res |= RADEON_PB_USAGE_CACHE;
390
391    if (domain & R300_DOMAIN_GTT)
392        res |= RADEON_PB_USAGE_DOMAIN_GTT;
393
394    if (domain & R300_DOMAIN_VRAM)
395        res |= RADEON_PB_USAGE_DOMAIN_VRAM;
396
397    return res;
398}
399
400static struct r300_winsys_bo *
401radeon_winsys_bo_create(struct r300_winsys_screen *rws,
402                        unsigned size,
403                        unsigned alignment,
404                        unsigned bind,
405                        unsigned usage,
406                        enum r300_buffer_domain domain)
407{
408    struct radeon_drm_winsys *ws = radeon_drm_winsys(rws);
409    struct pb_desc desc;
410    struct pb_manager *provider;
411    struct pb_buffer *buffer;
412
413    memset(&desc, 0, sizeof(desc));
414    desc.alignment = alignment;
415    desc.usage = get_pb_usage_from_create_flags(bind, usage, domain);
416
417    /* Assign a buffer manager. */
418    if (bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
419	provider = ws->cman;
420    else
421        provider = ws->kman;
422
423    buffer = provider->create_buffer(provider, size, &desc);
424    if (!buffer)
425	return NULL;
426
427    return (struct r300_winsys_bo*)buffer;
428}
429
430static struct r300_winsys_bo *radeon_winsys_bo_from_handle(struct r300_winsys_screen *rws,
431                                                           struct winsys_handle *whandle,
432                                                           unsigned *stride,
433                                                           unsigned *size)
434{
435    struct radeon_drm_winsys *ws = radeon_drm_winsys(rws);
436    struct radeon_bo *bo;
437    struct radeon_bomgr *mgr = radeon_bomgr(ws->kman);
438    struct drm_gem_open open_arg = {};
439
440    /* We must maintain a list of pairs <handle, bo>, so that we always return
441     * the same BO for one particular handle. If we didn't do that and created
442     * more than one BO for the same handle and then relocated them in a CS,
443     * we would hit a deadlock in the kernel.
444     *
445     * The list of pairs is guarded by a mutex, of course. */
446    pipe_mutex_lock(mgr->bo_handles_mutex);
447
448    /* First check if there already is an existing bo for the handle. */
449    bo = util_hash_table_get(mgr->bo_handles, (void*)(uintptr_t)whandle->handle);
450    if (bo) {
451        /* Increase the refcount. */
452        struct pb_buffer *b = NULL;
453        pb_reference(&b, &bo->base);
454        goto done;
455    }
456
457    /* There isn't, create a new one. */
458    bo = CALLOC_STRUCT(radeon_bo);
459    if (!bo) {
460        goto fail;
461    }
462
463    /* Open the BO. */
464    open_arg.name = whandle->handle;
465    if (drmIoctl(ws->fd, DRM_IOCTL_GEM_OPEN, &open_arg)) {
466        FREE(bo);
467        goto fail;
468    }
469    bo->handle = open_arg.handle;
470    bo->size = open_arg.size;
471    bo->name = whandle->handle;
472    radeon_bo_ref(bo);
473
474    /* Initialize it. */
475    pipe_reference_init(&bo->base.base.reference, 1);
476    bo->base.base.alignment = 0;
477    bo->base.base.usage = PB_USAGE_GPU_WRITE | PB_USAGE_GPU_READ;
478    bo->base.base.size = bo->size;
479    bo->base.vtbl = &radeon_bo_vtbl;
480    bo->mgr = mgr;
481
482    util_hash_table_set(mgr->bo_handles, (void*)(uintptr_t)whandle->handle, bo);
483
484done:
485    pipe_mutex_unlock(mgr->bo_handles_mutex);
486
487    if (stride)
488        *stride = whandle->stride;
489    if (size)
490        *size = bo->base.base.size;
491
492    return (struct r300_winsys_bo*)bo;
493
494fail:
495    pipe_mutex_unlock(mgr->bo_handles_mutex);
496    return NULL;
497}
498
499static boolean radeon_winsys_bo_get_handle(struct r300_winsys_bo *buffer,
500                                           unsigned stride,
501                                           struct winsys_handle *whandle)
502{
503    struct drm_gem_flink flink = {};
504    struct radeon_bo *bo = get_radeon_bo(pb_buffer(buffer));
505    whandle->stride = stride;
506
507
508    if (whandle->type == DRM_API_HANDLE_TYPE_SHARED) {
509        if (!bo->flinked) {
510            flink.handle = bo->handle;
511
512            if (ioctl(bo->mgr->rws->fd, DRM_IOCTL_GEM_FLINK, &flink)) {
513                return FALSE;
514            }
515
516            bo->flinked = TRUE;
517            bo->flink = flink.name;
518        }
519        whandle->handle = bo->flink;
520    } else if (whandle->type == DRM_API_HANDLE_TYPE_KMS) {
521        whandle->handle = bo->handle;
522    }
523    return TRUE;
524}
525
526void radeon_bomgr_init_functions(struct radeon_drm_winsys *ws)
527{
528    ws->base.buffer_get_cs_handle = radeon_drm_get_cs_handle;
529    ws->base.buffer_set_tiling = radeon_bo_set_tiling;
530    ws->base.buffer_get_tiling = radeon_bo_get_tiling;
531    ws->base.buffer_map = radeon_bo_map;
532    ws->base.buffer_unmap = pb_unmap;
533    ws->base.buffer_wait = radeon_bo_wait;
534    ws->base.buffer_is_busy = radeon_bo_is_busy;
535    ws->base.buffer_create = radeon_winsys_bo_create;
536    ws->base.buffer_from_handle = radeon_winsys_bo_from_handle;
537    ws->base.buffer_get_handle = radeon_winsys_bo_get_handle;
538}
539