1/*
2 *  Copyright (C) 2007 OpenedHand
3 *  Copyright (C) 2007 Alp Toker <alp@atoker.com>
4 *
5 *  This library is free software; you can redistribute it and/or
6 *  modify it under the terms of the GNU Lesser General Public
7 *  License as published by the Free Software Foundation; either
8 *  version 2 of the License, or (at your option) any later version.
9 *
10 *  This library is distributed in the hope that it will be useful,
11 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
12 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 *  Lesser General Public License for more details.
14 *
15 *  You should have received a copy of the GNU Lesser General Public
16 *  License along with this library; if not, write to the Free Software
17 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
18 */
19
20/**
21 * SECTION:webkit-video-sink
22 * @short_description: GStreamer video sink
23 *
24 * #WebKitVideoSink is a GStreamer sink element that triggers
25 * repaints in the WebKit GStreamer media player for the
26 * current video buffer.
27 */
28
29#include "config.h"
30#include "VideoSinkGStreamer.h"
31#if USE(GSTREAMER)
32
33#include <glib.h>
34#include <gst/gst.h>
35#include <gst/video/video.h>
36
37static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE("sink",
38                                                                   GST_PAD_SINK, GST_PAD_ALWAYS,
39// CAIRO_FORMAT_RGB24 used to render the video buffers is little/big endian dependant.
40#if G_BYTE_ORDER == G_LITTLE_ENDIAN
41                                                                   GST_STATIC_CAPS(GST_VIDEO_CAPS_BGRx ";" GST_VIDEO_CAPS_BGRA)
42#else
43                                                                   GST_STATIC_CAPS(GST_VIDEO_CAPS_xRGB ";" GST_VIDEO_CAPS_ARGB)
44#endif
45);
46
47GST_DEBUG_CATEGORY_STATIC(webkit_video_sink_debug);
48#define GST_CAT_DEFAULT webkit_video_sink_debug
49
50enum {
51    REPAINT_REQUESTED,
52    LAST_SIGNAL
53};
54
55enum {
56    PROP_0
57};
58
59static guint webkit_video_sink_signals[LAST_SIGNAL] = { 0, };
60
61struct _WebKitVideoSinkPrivate {
62    GstBuffer* buffer;
63    guint timeout_id;
64    GMutex* buffer_mutex;
65    GCond* data_cond;
66
67    // If this is TRUE all processing should finish ASAP
68    // This is necessary because there could be a race between
69    // unlock() and render(), where unlock() wins, signals the
70    // GCond, then render() tries to render a frame although
71    // everything else isn't running anymore. This will lead
72    // to deadlocks because render() holds the stream lock.
73    //
74    // Protected by the buffer mutex
75    gboolean unlocked;
76};
77
78#define _do_init(bla) \
79    GST_DEBUG_CATEGORY_INIT(webkit_video_sink_debug, \
80                            "webkitsink", \
81                            0, \
82                            "webkit video sink")
83
84GST_BOILERPLATE_FULL(WebKitVideoSink,
85                     webkit_video_sink,
86                     GstVideoSink,
87                     GST_TYPE_VIDEO_SINK,
88                     _do_init);
89
90static void
91webkit_video_sink_base_init(gpointer g_class)
92{
93    GstElementClass* element_class = GST_ELEMENT_CLASS(g_class);
94
95    gst_element_class_add_pad_template(element_class, gst_static_pad_template_get(&sinktemplate));
96    gst_element_class_set_details_simple(element_class, "WebKit video sink",
97            "Sink/Video", "Sends video data from a GStreamer pipeline to a Cairo surface",
98            "Alp Toker <alp@atoker.com>");
99}
100
101static void
102webkit_video_sink_init(WebKitVideoSink* sink, WebKitVideoSinkClass* klass)
103{
104    WebKitVideoSinkPrivate* priv;
105
106    sink->priv = priv = G_TYPE_INSTANCE_GET_PRIVATE(sink, WEBKIT_TYPE_VIDEO_SINK, WebKitVideoSinkPrivate);
107    priv->data_cond = g_cond_new();
108    priv->buffer_mutex = g_mutex_new();
109}
110
111static gboolean
112webkit_video_sink_timeout_func(gpointer data)
113{
114    WebKitVideoSink* sink = reinterpret_cast<WebKitVideoSink*>(data);
115    WebKitVideoSinkPrivate* priv = sink->priv;
116    GstBuffer* buffer;
117
118    g_mutex_lock(priv->buffer_mutex);
119    buffer = priv->buffer;
120    priv->buffer = 0;
121    priv->timeout_id = 0;
122
123    if (!buffer || priv->unlocked || G_UNLIKELY(!GST_IS_BUFFER(buffer))) {
124        g_cond_signal(priv->data_cond);
125        g_mutex_unlock(priv->buffer_mutex);
126        return FALSE;
127    }
128
129    g_signal_emit(sink, webkit_video_sink_signals[REPAINT_REQUESTED], 0, buffer);
130    gst_buffer_unref(buffer);
131    g_cond_signal(priv->data_cond);
132    g_mutex_unlock(priv->buffer_mutex);
133
134    return FALSE;
135}
136
137static GstFlowReturn
138webkit_video_sink_render(GstBaseSink* bsink, GstBuffer* buffer)
139{
140    WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(bsink);
141    WebKitVideoSinkPrivate* priv = sink->priv;
142
143    g_mutex_lock(priv->buffer_mutex);
144
145    if (priv->unlocked) {
146        g_mutex_unlock(priv->buffer_mutex);
147        return GST_FLOW_OK;
148    }
149
150    priv->buffer = gst_buffer_ref(buffer);
151
152    // For the unlikely case where the buffer has no caps, the caps
153    // are implicitely the caps of the pad. This shouldn't happen.
154    if (G_UNLIKELY(!GST_BUFFER_CAPS(buffer))) {
155        buffer = priv->buffer = gst_buffer_make_metadata_writable(priv->buffer);
156        gst_buffer_set_caps(priv->buffer, GST_PAD_CAPS(GST_BASE_SINK_PAD(bsink)));
157    }
158
159    GstCaps *caps = GST_BUFFER_CAPS(buffer);
160    GstVideoFormat format;
161    int width, height;
162    if (G_UNLIKELY(!gst_video_format_parse_caps(caps, &format, &width, &height))) {
163        gst_buffer_unref(buffer);
164        g_mutex_unlock(priv->buffer_mutex);
165        return GST_FLOW_ERROR;
166    }
167
168    // Cairo's ARGB has pre-multiplied alpha while GStreamer's doesn't.
169    // Here we convert to Cairo's ARGB.
170    if (format == GST_VIDEO_FORMAT_ARGB || format == GST_VIDEO_FORMAT_BGRA) {
171        // Because GstBaseSink::render() only owns the buffer reference in the
172        // method scope we can't use gst_buffer_make_writable() here. Also
173        // The buffer content should not be changed here because the same buffer
174        // could be passed multiple times to this method (in theory)
175        GstBuffer *newBuffer = gst_buffer_try_new_and_alloc(GST_BUFFER_SIZE(buffer));
176
177        // Check if allocation failed
178        if (G_UNLIKELY(!newBuffer)) {
179            gst_buffer_unref(buffer);
180            g_mutex_unlock(priv->buffer_mutex);
181            return GST_FLOW_ERROR;
182        }
183
184        gst_buffer_copy_metadata(newBuffer, buffer, (GstBufferCopyFlags) GST_BUFFER_COPY_ALL);
185
186        // We don't use Color::premultipliedARGBFromColor() here because
187        // one function call per video pixel is just too expensive:
188        // For 720p/PAL for example this means 1280*720*25=23040000
189        // function calls per second!
190        unsigned short alpha;
191        const guint8 *source = GST_BUFFER_DATA(buffer);
192        guint8 *destination = GST_BUFFER_DATA(newBuffer);
193
194        for (int x = 0; x < height; x++) {
195            for (int y = 0; y < width; y++) {
196#if G_BYTE_ORDER == G_LITTLE_ENDIAN
197                alpha = source[3];
198                destination[0] = (source[0] * alpha + 128) / 255;
199                destination[1] = (source[1] * alpha + 128) / 255;
200                destination[2] = (source[2] * alpha + 128) / 255;
201                destination[3] = alpha;
202#else
203                alpha = source[0];
204                destination[0] = alpha;
205                destination[1] = (source[1] * alpha + 128) / 255;
206                destination[2] = (source[2] * alpha + 128) / 255;
207                destination[3] = (source[3] * alpha + 128) / 255;
208#endif
209                source += 4;
210                destination += 4;
211            }
212        }
213        gst_buffer_unref(buffer);
214        buffer = priv->buffer = newBuffer;
215    }
216
217    // This should likely use a lower priority, but glib currently starves
218    // lower priority sources.
219    // See: https://bugzilla.gnome.org/show_bug.cgi?id=610830.
220    priv->timeout_id = g_timeout_add_full(G_PRIORITY_DEFAULT, 0,
221                                          webkit_video_sink_timeout_func,
222                                          gst_object_ref(sink),
223                                          (GDestroyNotify)gst_object_unref);
224
225    g_cond_wait(priv->data_cond, priv->buffer_mutex);
226    g_mutex_unlock(priv->buffer_mutex);
227    return GST_FLOW_OK;
228}
229
230static void
231webkit_video_sink_dispose(GObject* object)
232{
233    WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object);
234    WebKitVideoSinkPrivate* priv = sink->priv;
235
236    if (priv->data_cond) {
237        g_cond_free(priv->data_cond);
238        priv->data_cond = 0;
239    }
240
241    if (priv->buffer_mutex) {
242        g_mutex_free(priv->buffer_mutex);
243        priv->buffer_mutex = 0;
244    }
245
246    G_OBJECT_CLASS(parent_class)->dispose(object);
247}
248
249static void
250unlock_buffer_mutex(WebKitVideoSinkPrivate* priv)
251{
252    g_mutex_lock(priv->buffer_mutex);
253
254    if (priv->buffer) {
255        gst_buffer_unref(priv->buffer);
256        priv->buffer = 0;
257    }
258
259    priv->unlocked = TRUE;
260
261    g_cond_signal(priv->data_cond);
262    g_mutex_unlock(priv->buffer_mutex);
263}
264
265static gboolean
266webkit_video_sink_unlock(GstBaseSink* object)
267{
268    WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object);
269
270    unlock_buffer_mutex(sink->priv);
271
272    return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock,
273                                        (object), TRUE);
274}
275
276static gboolean
277webkit_video_sink_unlock_stop(GstBaseSink* object)
278{
279    WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object);
280    WebKitVideoSinkPrivate* priv = sink->priv;
281
282    g_mutex_lock(priv->buffer_mutex);
283    priv->unlocked = FALSE;
284    g_mutex_unlock(priv->buffer_mutex);
285
286    return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock_stop,
287                                        (object), TRUE);
288}
289
290static gboolean
291webkit_video_sink_stop(GstBaseSink* base_sink)
292{
293    WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(base_sink)->priv;
294
295    unlock_buffer_mutex(priv);
296    return TRUE;
297}
298
299static gboolean
300webkit_video_sink_start(GstBaseSink* base_sink)
301{
302    WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(base_sink)->priv;
303
304    g_mutex_lock(priv->buffer_mutex);
305    priv->unlocked = FALSE;
306    g_mutex_unlock(priv->buffer_mutex);
307    return TRUE;
308}
309
310static void
311marshal_VOID__MINIOBJECT(GClosure * closure, GValue * return_value,
312                         guint n_param_values, const GValue * param_values,
313                         gpointer invocation_hint, gpointer marshal_data)
314{
315  typedef void (*marshalfunc_VOID__MINIOBJECT) (gpointer obj, gpointer arg1, gpointer data2);
316  marshalfunc_VOID__MINIOBJECT callback;
317  GCClosure *cc = (GCClosure *) closure;
318  gpointer data1, data2;
319
320  g_return_if_fail(n_param_values == 2);
321
322  if (G_CCLOSURE_SWAP_DATA(closure)) {
323      data1 = closure->data;
324      data2 = g_value_peek_pointer(param_values + 0);
325  } else {
326      data1 = g_value_peek_pointer(param_values + 0);
327      data2 = closure->data;
328  }
329  callback = (marshalfunc_VOID__MINIOBJECT) (marshal_data ? marshal_data : cc->callback);
330
331  callback(data1, gst_value_get_mini_object(param_values + 1), data2);
332}
333
334static void
335webkit_video_sink_class_init(WebKitVideoSinkClass* klass)
336{
337    GObjectClass* gobject_class = G_OBJECT_CLASS(klass);
338    GstBaseSinkClass* gstbase_sink_class = GST_BASE_SINK_CLASS(klass);
339
340    g_type_class_add_private(klass, sizeof(WebKitVideoSinkPrivate));
341
342    gobject_class->dispose = webkit_video_sink_dispose;
343
344    gstbase_sink_class->unlock = webkit_video_sink_unlock;
345    gstbase_sink_class->unlock_stop = webkit_video_sink_unlock_stop;
346    gstbase_sink_class->render = webkit_video_sink_render;
347    gstbase_sink_class->preroll = webkit_video_sink_render;
348    gstbase_sink_class->stop = webkit_video_sink_stop;
349    gstbase_sink_class->start = webkit_video_sink_start;
350
351    webkit_video_sink_signals[REPAINT_REQUESTED] = g_signal_new("repaint-requested",
352            G_TYPE_FROM_CLASS(klass),
353            (GSignalFlags)(G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
354            0,
355            0,
356            0,
357            marshal_VOID__MINIOBJECT,
358            G_TYPE_NONE, 1, GST_TYPE_BUFFER);
359}
360
361/**
362 * webkit_video_sink_new:
363 *
364 * Creates a new GStreamer video sink.
365 *
366 * Return value: a #GstElement for the newly created video sink
367 */
368GstElement*
369webkit_video_sink_new(void)
370{
371    return (GstElement*)g_object_new(WEBKIT_TYPE_VIDEO_SINK, 0);
372}
373
374#endif // USE(GSTREAMER)
375