1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "media/filters/skcanvas_video_renderer.h"
6
7#include "base/logging.h"
8#include "media/base/video_frame.h"
9#include "media/base/yuv_convert.h"
10#include "third_party/libyuv/include/libyuv.h"
11#include "third_party/skia/include/core/SkCanvas.h"
12#include "third_party/skia/include/core/SkImageGenerator.h"
13#include "ui/gfx/skbitmap_operations.h"
14
15// Skia internal format depends on a platform. On Android it is ABGR, on others
16// it is ARGB.
17#if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \
18    SK_A32_SHIFT == 24
19#define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB
20#define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB
21#elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \
22    SK_A32_SHIFT == 24
23#define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR
24#define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR
25#else
26#error Unexpected Skia ARGB_8888 layout!
27#endif
28
29namespace media {
30
31static bool IsYUV(media::VideoFrame::Format format) {
32  switch (format) {
33    case VideoFrame::YV12:
34    case VideoFrame::YV16:
35    case VideoFrame::I420:
36    case VideoFrame::YV12A:
37    case VideoFrame::YV12J:
38    case VideoFrame::YV24:
39    case VideoFrame::NV12:
40      return true;
41    case VideoFrame::UNKNOWN:
42    case VideoFrame::NATIVE_TEXTURE:
43#if defined(VIDEO_HOLE)
44    case VideoFrame::HOLE:
45#endif  // defined(VIDEO_HOLE)
46      return false;
47  }
48  NOTREACHED() << "Invalid videoframe format provided: " << format;
49  return false;
50}
51
52static bool IsJPEGColorSpace(media::VideoFrame::Format format) {
53  switch (format) {
54    case VideoFrame::YV12J:
55      return true;
56    case VideoFrame::YV12:
57    case VideoFrame::YV16:
58    case VideoFrame::I420:
59    case VideoFrame::YV12A:
60    case VideoFrame::YV24:
61    case VideoFrame::NV12:
62    case VideoFrame::UNKNOWN:
63    case VideoFrame::NATIVE_TEXTURE:
64#if defined(VIDEO_HOLE)
65    case VideoFrame::HOLE:
66#endif  // defined(VIDEO_HOLE)
67      return false;
68  }
69  NOTREACHED() << "Invalid videoframe format provided: " << format;
70  return false;
71}
72
73static bool IsYUVOrNative(media::VideoFrame::Format format) {
74  return IsYUV(format) || format == media::VideoFrame::NATIVE_TEXTURE;
75}
76
77// Converts a |video_frame| to raw |rgb_pixels|.
78static void ConvertVideoFrameToRGBPixels(
79    const scoped_refptr<media::VideoFrame>& video_frame,
80    void* rgb_pixels,
81    size_t row_bytes) {
82  DCHECK(IsYUVOrNative(video_frame->format()))
83      << video_frame->format();
84  if (IsYUV(video_frame->format())) {
85    DCHECK_EQ(video_frame->stride(media::VideoFrame::kUPlane),
86              video_frame->stride(media::VideoFrame::kVPlane));
87  }
88
89  size_t y_offset = 0;
90  size_t uv_offset = 0;
91  if (IsYUV(video_frame->format())) {
92    int y_shift = (video_frame->format() == media::VideoFrame::YV16) ? 0 : 1;
93    // Use the "left" and "top" of the destination rect to locate the offset
94    // in Y, U and V planes.
95    y_offset = (video_frame->stride(media::VideoFrame::kYPlane) *
96                video_frame->visible_rect().y()) +
97                video_frame->visible_rect().x();
98    // For format YV12, there is one U, V value per 2x2 block.
99    // For format YV16, there is one U, V value per 2x1 block.
100    uv_offset = (video_frame->stride(media::VideoFrame::kUPlane) *
101                (video_frame->visible_rect().y() >> y_shift)) +
102                (video_frame->visible_rect().x() >> 1);
103  }
104
105  switch (video_frame->format()) {
106    case media::VideoFrame::YV12:
107    case media::VideoFrame::I420:
108      LIBYUV_I420_TO_ARGB(
109          video_frame->data(media::VideoFrame::kYPlane) + y_offset,
110          video_frame->stride(media::VideoFrame::kYPlane),
111          video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
112          video_frame->stride(media::VideoFrame::kUPlane),
113          video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
114          video_frame->stride(media::VideoFrame::kVPlane),
115          static_cast<uint8*>(rgb_pixels),
116          row_bytes,
117          video_frame->visible_rect().width(),
118          video_frame->visible_rect().height());
119      break;
120
121    case media::VideoFrame::YV12J:
122      media::ConvertYUVToRGB32(
123          video_frame->data(media::VideoFrame::kYPlane) + y_offset,
124          video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
125          video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
126          static_cast<uint8*>(rgb_pixels),
127          video_frame->visible_rect().width(),
128          video_frame->visible_rect().height(),
129          video_frame->stride(media::VideoFrame::kYPlane),
130          video_frame->stride(media::VideoFrame::kUPlane),
131          row_bytes,
132          media::YV12J);
133      break;
134
135    case media::VideoFrame::YV16:
136      LIBYUV_I422_TO_ARGB(
137          video_frame->data(media::VideoFrame::kYPlane) + y_offset,
138          video_frame->stride(media::VideoFrame::kYPlane),
139          video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
140          video_frame->stride(media::VideoFrame::kUPlane),
141          video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
142          video_frame->stride(media::VideoFrame::kVPlane),
143          static_cast<uint8*>(rgb_pixels),
144          row_bytes,
145          video_frame->visible_rect().width(),
146          video_frame->visible_rect().height());
147      break;
148
149    case media::VideoFrame::YV12A:
150      // Since libyuv doesn't support YUVA, fallback to media, which is not ARM
151      // optimized.
152      // TODO(fbarchard, mtomasz): Use libyuv, then copy the alpha channel.
153      media::ConvertYUVAToARGB(
154          video_frame->data(media::VideoFrame::kYPlane) + y_offset,
155          video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
156          video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
157          video_frame->data(media::VideoFrame::kAPlane),
158          static_cast<uint8*>(rgb_pixels),
159          video_frame->visible_rect().width(),
160          video_frame->visible_rect().height(),
161          video_frame->stride(media::VideoFrame::kYPlane),
162          video_frame->stride(media::VideoFrame::kUPlane),
163          video_frame->stride(media::VideoFrame::kAPlane),
164          row_bytes,
165          media::YV12);
166      break;
167
168    case media::VideoFrame::YV24:
169      libyuv::I444ToARGB(
170          video_frame->data(media::VideoFrame::kYPlane) + y_offset,
171          video_frame->stride(media::VideoFrame::kYPlane),
172          video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
173          video_frame->stride(media::VideoFrame::kUPlane),
174          video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
175          video_frame->stride(media::VideoFrame::kVPlane),
176          static_cast<uint8*>(rgb_pixels),
177          row_bytes,
178          video_frame->visible_rect().width(),
179          video_frame->visible_rect().height());
180#if SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \
181    SK_A32_SHIFT == 24
182      libyuv::ARGBToABGR(static_cast<uint8*>(rgb_pixels),
183                         row_bytes,
184                         static_cast<uint8*>(rgb_pixels),
185                         row_bytes,
186                         video_frame->visible_rect().width(),
187                         video_frame->visible_rect().height());
188#endif
189      break;
190
191    case media::VideoFrame::NATIVE_TEXTURE: {
192      DCHECK_EQ(video_frame->format(), media::VideoFrame::NATIVE_TEXTURE);
193      SkBitmap tmp;
194      tmp.installPixels(
195          SkImageInfo::MakeN32Premul(video_frame->visible_rect().width(),
196                                     video_frame->visible_rect().height()),
197          rgb_pixels,
198          row_bytes);
199      video_frame->ReadPixelsFromNativeTexture(tmp);
200      break;
201    }
202    default:
203      NOTREACHED();
204      break;
205  }
206}
207
208// Generates an RGB image from a VideoFrame.
209class VideoImageGenerator : public SkImageGenerator {
210 public:
211  VideoImageGenerator(const scoped_refptr<VideoFrame>& frame) : frame_(frame) {}
212  virtual ~VideoImageGenerator() {}
213
214  void set_frame(const scoped_refptr<VideoFrame>& frame) { frame_ = frame; }
215
216 protected:
217  virtual bool onGetInfo(SkImageInfo* info) OVERRIDE {
218    info->fWidth = frame_->visible_rect().width();
219    info->fHeight = frame_->visible_rect().height();
220    info->fColorType = kN32_SkColorType;
221    info->fAlphaType = kPremul_SkAlphaType;
222    return true;
223  }
224
225  virtual bool onGetPixels(const SkImageInfo& info,
226                           void* pixels,
227                           size_t row_bytes,
228                           SkPMColor ctable[],
229                           int* ctable_count) OVERRIDE {
230    if (!frame_.get())
231      return false;
232    if (!pixels)
233      return true;
234    // If skia couldn't do the YUV conversion, we will.
235    ConvertVideoFrameToRGBPixels(frame_, pixels, row_bytes);
236    frame_ = NULL;
237    return true;
238  }
239
240  virtual bool onGetYUV8Planes(SkISize sizes[3],
241                               void* planes[3],
242                               size_t row_bytes[3],
243                               SkYUVColorSpace* color_space) OVERRIDE {
244    if (!frame_.get() || !IsYUV(frame_->format()))
245      return false;
246
247    if (color_space) {
248      if (IsJPEGColorSpace(frame_->format()))
249        *color_space = kJPEG_SkYUVColorSpace;
250      else
251        *color_space = kRec601_SkYUVColorSpace;
252    }
253
254    for (int plane = VideoFrame::kYPlane; plane <= VideoFrame::kVPlane;
255         ++plane) {
256      if (sizes) {
257        gfx::Size size;
258        size =
259            VideoFrame::PlaneSize(frame_->format(),
260                                  plane,
261                                  gfx::Size(frame_->visible_rect().width(),
262                                            frame_->visible_rect().height()));
263        sizes[plane].set(size.width(), size.height());
264      }
265      if (row_bytes && planes) {
266        size_t offset;
267        int y_shift = (frame_->format() == media::VideoFrame::YV16) ? 0 : 1;
268        if (plane == media::VideoFrame::kYPlane) {
269          offset = (frame_->stride(media::VideoFrame::kYPlane) *
270                    frame_->visible_rect().y()) +
271                   frame_->visible_rect().x();
272        } else {
273          offset = (frame_->stride(media::VideoFrame::kUPlane) *
274                    (frame_->visible_rect().y() >> y_shift)) +
275                   (frame_->visible_rect().x() >> 1);
276        }
277        row_bytes[plane] = static_cast<size_t>(frame_->stride(plane));
278        planes[plane] = frame_->data(plane) + offset;
279      }
280    }
281    if (planes && row_bytes)
282      frame_ = NULL;
283    return true;
284  }
285
286 private:
287  scoped_refptr<VideoFrame> frame_;
288};
289
290SkCanvasVideoRenderer::SkCanvasVideoRenderer()
291    : generator_(NULL), last_frame_timestamp_(media::kNoTimestamp()) {
292  last_frame_.setIsVolatile(true);
293}
294
295SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {}
296
297void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
298                                  SkCanvas* canvas,
299                                  const gfx::RectF& dest_rect,
300                                  uint8 alpha,
301                                  SkXfermode::Mode mode,
302                                  VideoRotation video_rotation) {
303  if (alpha == 0) {
304    return;
305  }
306
307  SkRect dest;
308  dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom());
309
310  SkPaint paint;
311  paint.setAlpha(alpha);
312
313  // Paint black rectangle if there isn't a frame available or the
314  // frame has an unexpected format.
315  if (!video_frame.get() || !IsYUVOrNative(video_frame->format())) {
316    canvas->drawRect(dest, paint);
317    canvas->flush();
318    return;
319  }
320
321  // Check if we should convert and update |last_frame_|.
322  if (last_frame_.isNull() ||
323      video_frame->timestamp() != last_frame_timestamp_) {
324    generator_ = new VideoImageGenerator(video_frame);
325
326    // Note: This takes ownership of |generator_|.
327    if (!SkInstallDiscardablePixelRef(generator_, &last_frame_)) {
328      NOTREACHED();
329    }
330
331    // TODO(rileya): Perform this rotation on the canvas, rather than allocating
332    // a new bitmap and copying.
333    switch (video_rotation) {
334      case VIDEO_ROTATION_0:
335        break;
336      case VIDEO_ROTATION_90:
337        last_frame_ = SkBitmapOperations::Rotate(
338            last_frame_, SkBitmapOperations::ROTATION_90_CW);
339        break;
340      case VIDEO_ROTATION_180:
341        last_frame_ = SkBitmapOperations::Rotate(
342            last_frame_, SkBitmapOperations::ROTATION_180_CW);
343        break;
344      case VIDEO_ROTATION_270:
345        last_frame_ = SkBitmapOperations::Rotate(
346            last_frame_, SkBitmapOperations::ROTATION_270_CW);
347        break;
348    }
349
350    // We copied the frame into a new bitmap and threw out the old one, so we
351    // no longer have a |generator_| around. This should be removed when the
352    // above TODO is addressed.
353    if (video_rotation != VIDEO_ROTATION_0)
354      generator_ = NULL;
355
356    last_frame_timestamp_ = video_frame->timestamp();
357  } else if (generator_) {
358    generator_->set_frame(video_frame);
359  }
360
361  paint.setXfermodeMode(mode);
362
363  // Paint using |last_frame_|.
364  paint.setFilterLevel(SkPaint::kLow_FilterLevel);
365  canvas->drawBitmapRect(last_frame_, NULL, dest, &paint);
366  canvas->flush();
367}
368
369void SkCanvasVideoRenderer::Copy(const scoped_refptr<VideoFrame>& video_frame,
370                                 SkCanvas* canvas) {
371  Paint(video_frame,
372        canvas,
373        video_frame->visible_rect(),
374        0xff,
375        SkXfermode::kSrc_Mode,
376        media::VIDEO_ROTATION_0);
377}
378
379}  // namespace media
380