1/**************************************************************************
2 *
3 * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA
4 * All Rights Reserved.
5 * Copyright 2009 VMware, Inc., Palo Alto, CA., USA
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
23 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
24 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
25 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
26 * USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
28 **************************************************************************/
29/*
30 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
31 */
32
33#ifdef HAVE_CONFIG_H
34#include "config.h"
35#endif
36
37#include "wsbm_fencemgr.h"
38#include "wsbm_pool.h"
39#include "wsbm_manager.h"
40#include <xf86drm.h>
41#include <drm/psb_ttm_fence_user.h>
42#include <string.h>
43#include <unistd.h>
44
45struct _WsbmFenceClass
46{
47    struct _WsbmListHead head;
48    struct _WsbmMutex mutex;
49    struct _WsbmMutex cmd_mutex;
50};
51
52/*
53 * Note: The struct _WsbmFenceMgr::Mutex should never be held
54 * during sleeps, since that may block fast concurrent access to
55 * fence data.
56 */
57
58struct _WsbmFenceMgr
59{
60    /*
61     * Constant members. Need no mutex protection.
62     */
63    struct _WsbmFenceMgrCreateInfo info;
64    void *private;
65
66    /*
67     * Atomic members. No mutex protection.
68     */
69
70    struct _WsbmAtomic count;
71
72    /*
73     * These members are protected by this->mutex
74     */
75
76    struct _WsbmFenceClass *classes;
77    uint32_t num_classes;
78};
79
80struct _WsbmFenceObject
81{
82
83    /*
84     * These members are constant and need no mutex protection.
85     * Note that @private may point to a structure with its own
86     * mutex protection, that we don't care about.
87     */
88
89    struct _WsbmFenceMgr *mgr;
90    uint32_t fence_class;
91    uint32_t fence_type;
92    void *private;
93
94    /*
95     * Atomic members. No mutex protection. note that
96     * @signaled types is updated using a compare-and-swap
97     * scheme to guarantee atomicity.
98     */
99
100    struct _WsbmAtomic refCount;
101    struct _WsbmAtomic signaled_types;
102
103    /*
104     * These members are protected by mgr->mutex.
105     */
106    struct _WsbmListHead head;
107};
108
109uint32_t
110wsbmFenceType(struct _WsbmFenceObject *fence)
111{
112    return fence->fence_type;
113}
114
115struct _WsbmFenceMgr *
116wsbmFenceMgrCreate(const struct _WsbmFenceMgrCreateInfo *info)
117{
118    struct _WsbmFenceMgr *tmp;
119    uint32_t i, j;
120    int ret;
121
122    tmp = calloc(1, sizeof(*tmp));
123    if (!tmp)
124	return NULL;
125
126    tmp->info = *info;
127    tmp->classes = calloc(tmp->info.num_classes, sizeof(*tmp->classes));
128    if (!tmp->classes)
129	goto out_err;
130
131    for (i = 0; i < tmp->info.num_classes; ++i) {
132	struct _WsbmFenceClass *fc = &tmp->classes[i];
133
134	WSBMINITLISTHEAD(&fc->head);
135	ret = WSBM_MUTEX_INIT(&fc->mutex);
136	if (ret)
137	    goto out_err1;
138	ret = WSBM_MUTEX_INIT(&fc->cmd_mutex);
139	if (ret) {
140	    WSBM_MUTEX_FREE(&fc->mutex);
141	    goto out_err1;
142	}
143    }
144    wsbmAtomicSet(&tmp->count, 0);
145
146    return tmp;
147
148  out_err1:
149    for (j = 0; j < i; ++j) {
150	WSBM_MUTEX_FREE(&tmp->classes[j].mutex);
151	WSBM_MUTEX_FREE(&tmp->classes[j].cmd_mutex);
152    }
153    free(tmp->classes);
154  out_err:
155    if (tmp)
156	free(tmp);
157    return NULL;
158}
159
160void
161wsbmFenceUnreference(struct _WsbmFenceObject **pFence)
162{
163    struct _WsbmFenceObject *fence = *pFence;
164    struct _WsbmFenceMgr *mgr;
165
166    *pFence = NULL;
167    if (fence == NULL)
168	return;
169
170    mgr = fence->mgr;
171    if (wsbmAtomicDecZero(&fence->refCount)) {
172	struct _WsbmFenceClass *fc = &mgr->classes[fence->fence_class];
173
174	WSBM_MUTEX_LOCK(&fc->mutex);
175	WSBMLISTDELINIT(&fence->head);
176	WSBM_MUTEX_UNLOCK(&fc->mutex);
177	if (fence->private)
178	    mgr->info.unreference(mgr, &fence->private);
179	fence->mgr = NULL;
180	wsbmAtomicDecZero(&mgr->count);
181	free(fence);
182    }
183}
184
185static void
186wsbmSignalPreviousFences(struct _WsbmFenceMgr *mgr,
187			 struct _WsbmListHead *list,
188			 uint32_t fence_class, uint32_t signaled_types)
189{
190    struct _WsbmFenceClass *fc = &mgr->classes[fence_class];
191    struct _WsbmFenceObject *entry;
192    struct _WsbmListHead *prev;
193    uint32_t old_signaled_types;
194    uint32_t ret_st;
195
196    WSBM_MUTEX_LOCK(&fc->mutex);
197    while (list != &fc->head && list->next != list) {
198	entry = WSBMLISTENTRY(list, struct _WsbmFenceObject, head);
199
200	prev = list->prev;
201
202	do {
203	    old_signaled_types = wsbmAtomicRead(&entry->signaled_types);
204	    signaled_types =
205		old_signaled_types | (signaled_types & entry->fence_type);
206	    if (signaled_types == old_signaled_types)
207		break;
208
209	    ret_st =
210		wsbmAtomicCmpXchg(&entry->signaled_types, old_signaled_types,
211				  signaled_types);
212	} while (ret_st != old_signaled_types);
213
214	if (signaled_types == entry->fence_type)
215	    WSBMLISTDELINIT(list);
216
217	list = prev;
218    }
219    WSBM_MUTEX_UNLOCK(&fc->mutex);
220}
221
222int
223wsbmFenceFinish(struct _WsbmFenceObject *fence, uint32_t fence_type,
224		int lazy_hint)
225{
226    struct _WsbmFenceMgr *mgr = fence->mgr;
227    int ret = 0;
228
229    if ((wsbmAtomicRead(&fence->signaled_types) & fence_type) == fence_type)
230	goto out;
231
232    ret = mgr->info.finish(mgr, fence->private, fence_type, lazy_hint);
233    if (ret)
234	goto out;
235
236    wsbmSignalPreviousFences(mgr, &fence->head, fence->fence_class,
237			     fence_type);
238  out:
239    return ret;
240}
241
242uint32_t
243wsbmFenceSignaledTypeCached(struct _WsbmFenceObject * fence)
244{
245    return wsbmAtomicRead(&fence->signaled_types);
246}
247
248int
249wsbmFenceSignaledType(struct _WsbmFenceObject *fence, uint32_t flush_type,
250		      uint32_t * signaled)
251{
252    int ret = 0;
253    struct _WsbmFenceMgr *mgr;
254    uint32_t signaled_types;
255    uint32_t old_signaled_types;
256    uint32_t ret_st;
257
258    mgr = fence->mgr;
259    *signaled = wsbmAtomicRead(&fence->signaled_types);
260    if ((*signaled & flush_type) == flush_type)
261	goto out0;
262
263    ret = mgr->info.signaled(mgr, fence->private, flush_type, signaled);
264    if (ret) {
265	*signaled = wsbmAtomicRead(&fence->signaled_types);
266	goto out0;
267    }
268
269    do {
270	old_signaled_types = wsbmAtomicRead(&fence->signaled_types);
271	signaled_types = old_signaled_types | *signaled;
272	if (signaled_types == old_signaled_types)
273	    break;
274
275	ret_st = wsbmAtomicCmpXchg(&fence->signaled_types, old_signaled_types,
276				   signaled_types);
277	if (old_signaled_types == ret_st)
278	    wsbmSignalPreviousFences(mgr, &fence->head, fence->fence_class,
279				     *signaled);
280    } while (old_signaled_types != ret_st);
281
282    return 0;
283  out0:
284    return ret;
285}
286
287struct _WsbmFenceObject *
288wsbmFenceReference(struct _WsbmFenceObject *fence)
289{
290    if (fence == NULL)
291	return NULL;
292    wsbmAtomicInc(&fence->refCount);
293    return fence;
294}
295
296struct _WsbmFenceObject *
297wsbmFenceCreateSig(struct _WsbmFenceMgr *mgr, uint32_t fence_class,
298		   uint32_t fence_type, uint32_t signaled_types,
299		   void *private, size_t private_size)
300{
301    struct _WsbmFenceClass *fc = &mgr->classes[fence_class];
302    struct _WsbmFenceObject *fence;
303    size_t fence_size = sizeof(*fence);
304
305    if (private_size)
306	fence_size = ((fence_size + 15) & ~15);
307
308    fence = calloc(1, fence_size + private_size);
309
310    if (!fence)
311	goto out_err;
312
313    wsbmAtomicSet(&fence->refCount, 1);
314    fence->mgr = mgr;
315    fence->fence_class = fence_class;
316    fence->fence_type = fence_type;
317    wsbmAtomicSet(&fence->signaled_types, signaled_types);
318    fence->private = private;
319    if (private_size) {
320	fence->private = (void *)(((uint8_t *) fence) + fence_size);
321	memcpy(fence->private, private, private_size);
322    }
323
324    WSBM_MUTEX_LOCK(&fc->mutex);
325    WSBMLISTADDTAIL(&fence->head, &fc->head);
326    WSBM_MUTEX_UNLOCK(&fc->mutex);
327    wsbmAtomicInc(&mgr->count);
328    return fence;
329
330  out_err:
331    {
332	int ret = mgr->info.finish(mgr, private, fence_type, 0);
333
334	if (ret)
335	    usleep(10000000);
336    }
337    if (fence)
338	free(fence);
339
340    mgr->info.unreference(mgr, &private);
341    return NULL;
342}
343
344struct _WsbmFenceObject *
345wsbmFenceCreate(struct _WsbmFenceMgr *mgr, uint32_t fence_class,
346		uint32_t fence_type, void *private, size_t private_size)
347{
348  return wsbmFenceCreateSig(mgr, fence_class, fence_type, 0, private,
349			    private_size);
350}
351
352struct _WsbmTTMFenceMgrPriv
353{
354    int fd;
355    unsigned int devOffset;
356};
357
358static int
359tSignaled(struct _WsbmFenceMgr *mgr, void *private, uint32_t flush_type,
360	  uint32_t * signaled_type)
361{
362    struct _WsbmTTMFenceMgrPriv *priv =
363	(struct _WsbmTTMFenceMgrPriv *)mgr->private;
364    union ttm_fence_signaled_arg arg;
365    int ret;
366
367    arg.req.handle = (unsigned long)private;
368    arg.req.fence_type = flush_type;
369    arg.req.flush = 1;
370    *signaled_type = 0;
371
372    ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_SIGNALED,
373			      &arg, sizeof(arg));
374    if (ret)
375	return ret;
376
377    *signaled_type = arg.rep.signaled_types;
378    return 0;
379}
380
381static int
382tFinish(struct _WsbmFenceMgr *mgr, void *private, uint32_t fence_type,
383	int lazy_hint)
384{
385    struct _WsbmTTMFenceMgrPriv *priv =
386	(struct _WsbmTTMFenceMgrPriv *)mgr->private;
387    union ttm_fence_finish_arg arg =
388	{.req = {.handle = (unsigned long)private,
389		 .fence_type = fence_type,
390		 .mode = (lazy_hint) ? TTM_FENCE_FINISH_MODE_LAZY : 0}
391    };
392    int ret;
393
394    do {
395	ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_FINISH,
396				  &arg, sizeof(arg));
397    } while (ret == -EAGAIN || ret == -ERESTART);
398
399    return ret;
400}
401
402static int
403tUnref(struct _WsbmFenceMgr *mgr, void **private)
404{
405    struct _WsbmTTMFenceMgrPriv *priv =
406	(struct _WsbmTTMFenceMgrPriv *)mgr->private;
407    struct ttm_fence_unref_arg arg = {.handle = (unsigned long)*private };
408
409    *private = NULL;
410
411    return drmCommandWrite(priv->fd, priv->devOffset + TTM_FENCE_UNREF,
412			   &arg, sizeof(arg));
413}
414
415struct _WsbmFenceMgr *
416wsbmFenceMgrTTMInit(int fd, unsigned int numClass, unsigned int devOffset)
417{
418    struct _WsbmFenceMgrCreateInfo info;
419    struct _WsbmFenceMgr *mgr;
420    struct _WsbmTTMFenceMgrPriv *priv = malloc(sizeof(*priv));
421
422    if (!priv)
423	return NULL;
424
425    priv->fd = fd;
426    priv->devOffset = devOffset;
427
428    info.flags = WSBM_FENCE_CLASS_ORDERED;
429    info.num_classes = numClass;
430    info.signaled = tSignaled;
431    info.finish = tFinish;
432    info.unreference = tUnref;
433
434    mgr = wsbmFenceMgrCreate(&info);
435    if (mgr == NULL) {
436	free(priv);
437	return NULL;
438    }
439
440    mgr->private = (void *)priv;
441    return mgr;
442}
443
444void
445wsbmFenceCmdLock(struct _WsbmFenceMgr *mgr, uint32_t fence_class)
446{
447    WSBM_MUTEX_LOCK(&mgr->classes[fence_class].cmd_mutex);
448}
449
450void
451wsbmFenceCmdUnlock(struct _WsbmFenceMgr *mgr, uint32_t fence_class)
452{
453    WSBM_MUTEX_UNLOCK(&mgr->classes[fence_class].cmd_mutex);
454}
455
456void
457wsbmFenceMgrTTMTakedown(struct _WsbmFenceMgr *mgr)
458{
459    unsigned int i;
460
461    if (!mgr)
462	return;
463
464    if (mgr->private)
465	free(mgr->private);
466
467    for (i = 0; i < mgr->info.num_classes; ++i) {
468	WSBM_MUTEX_FREE(&mgr->classes[i].mutex);
469	WSBM_MUTEX_FREE(&mgr->classes[i].cmd_mutex);
470    }
471    free(mgr);
472
473    return;
474}
475