1// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "media/filters/skcanvas_video_renderer.h" 6 7#include "base/logging.h" 8#include "media/base/video_frame.h" 9#include "media/base/yuv_convert.h" 10#include "third_party/libyuv/include/libyuv.h" 11#include "third_party/skia/include/core/SkCanvas.h" 12#include "third_party/skia/include/core/SkImageGenerator.h" 13#include "ui/gfx/skbitmap_operations.h" 14 15// Skia internal format depends on a platform. On Android it is ABGR, on others 16// it is ARGB. 17#if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \ 18 SK_A32_SHIFT == 24 19#define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB 20#define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB 21#elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \ 22 SK_A32_SHIFT == 24 23#define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR 24#define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR 25#else 26#error Unexpected Skia ARGB_8888 layout! 27#endif 28 29namespace media { 30 31static bool IsYUV(media::VideoFrame::Format format) { 32 switch (format) { 33 case VideoFrame::YV12: 34 case VideoFrame::YV16: 35 case VideoFrame::I420: 36 case VideoFrame::YV12A: 37 case VideoFrame::YV12J: 38 case VideoFrame::YV24: 39 case VideoFrame::NV12: 40 return true; 41 case VideoFrame::UNKNOWN: 42 case VideoFrame::NATIVE_TEXTURE: 43#if defined(VIDEO_HOLE) 44 case VideoFrame::HOLE: 45#endif // defined(VIDEO_HOLE) 46 return false; 47 } 48 NOTREACHED() << "Invalid videoframe format provided: " << format; 49 return false; 50} 51 52static bool IsJPEGColorSpace(media::VideoFrame::Format format) { 53 switch (format) { 54 case VideoFrame::YV12J: 55 return true; 56 case VideoFrame::YV12: 57 case VideoFrame::YV16: 58 case VideoFrame::I420: 59 case VideoFrame::YV12A: 60 case VideoFrame::YV24: 61 case VideoFrame::NV12: 62 case VideoFrame::UNKNOWN: 63 case VideoFrame::NATIVE_TEXTURE: 64#if defined(VIDEO_HOLE) 65 case VideoFrame::HOLE: 66#endif // defined(VIDEO_HOLE) 67 return false; 68 } 69 NOTREACHED() << "Invalid videoframe format provided: " << format; 70 return false; 71} 72 73static bool IsYUVOrNative(media::VideoFrame::Format format) { 74 return IsYUV(format) || format == media::VideoFrame::NATIVE_TEXTURE; 75} 76 77// Converts a |video_frame| to raw |rgb_pixels|. 78static void ConvertVideoFrameToRGBPixels( 79 const scoped_refptr<media::VideoFrame>& video_frame, 80 void* rgb_pixels, 81 size_t row_bytes) { 82 DCHECK(IsYUVOrNative(video_frame->format())) 83 << video_frame->format(); 84 if (IsYUV(video_frame->format())) { 85 DCHECK_EQ(video_frame->stride(media::VideoFrame::kUPlane), 86 video_frame->stride(media::VideoFrame::kVPlane)); 87 } 88 89 size_t y_offset = 0; 90 size_t uv_offset = 0; 91 if (IsYUV(video_frame->format())) { 92 int y_shift = (video_frame->format() == media::VideoFrame::YV16) ? 0 : 1; 93 // Use the "left" and "top" of the destination rect to locate the offset 94 // in Y, U and V planes. 95 y_offset = (video_frame->stride(media::VideoFrame::kYPlane) * 96 video_frame->visible_rect().y()) + 97 video_frame->visible_rect().x(); 98 // For format YV12, there is one U, V value per 2x2 block. 99 // For format YV16, there is one U, V value per 2x1 block. 100 uv_offset = (video_frame->stride(media::VideoFrame::kUPlane) * 101 (video_frame->visible_rect().y() >> y_shift)) + 102 (video_frame->visible_rect().x() >> 1); 103 } 104 105 switch (video_frame->format()) { 106 case media::VideoFrame::YV12: 107 case media::VideoFrame::I420: 108 LIBYUV_I420_TO_ARGB( 109 video_frame->data(media::VideoFrame::kYPlane) + y_offset, 110 video_frame->stride(media::VideoFrame::kYPlane), 111 video_frame->data(media::VideoFrame::kUPlane) + uv_offset, 112 video_frame->stride(media::VideoFrame::kUPlane), 113 video_frame->data(media::VideoFrame::kVPlane) + uv_offset, 114 video_frame->stride(media::VideoFrame::kVPlane), 115 static_cast<uint8*>(rgb_pixels), 116 row_bytes, 117 video_frame->visible_rect().width(), 118 video_frame->visible_rect().height()); 119 break; 120 121 case media::VideoFrame::YV12J: 122 media::ConvertYUVToRGB32( 123 video_frame->data(media::VideoFrame::kYPlane) + y_offset, 124 video_frame->data(media::VideoFrame::kUPlane) + uv_offset, 125 video_frame->data(media::VideoFrame::kVPlane) + uv_offset, 126 static_cast<uint8*>(rgb_pixels), 127 video_frame->visible_rect().width(), 128 video_frame->visible_rect().height(), 129 video_frame->stride(media::VideoFrame::kYPlane), 130 video_frame->stride(media::VideoFrame::kUPlane), 131 row_bytes, 132 media::YV12J); 133 break; 134 135 case media::VideoFrame::YV16: 136 LIBYUV_I422_TO_ARGB( 137 video_frame->data(media::VideoFrame::kYPlane) + y_offset, 138 video_frame->stride(media::VideoFrame::kYPlane), 139 video_frame->data(media::VideoFrame::kUPlane) + uv_offset, 140 video_frame->stride(media::VideoFrame::kUPlane), 141 video_frame->data(media::VideoFrame::kVPlane) + uv_offset, 142 video_frame->stride(media::VideoFrame::kVPlane), 143 static_cast<uint8*>(rgb_pixels), 144 row_bytes, 145 video_frame->visible_rect().width(), 146 video_frame->visible_rect().height()); 147 break; 148 149 case media::VideoFrame::YV12A: 150 // Since libyuv doesn't support YUVA, fallback to media, which is not ARM 151 // optimized. 152 // TODO(fbarchard, mtomasz): Use libyuv, then copy the alpha channel. 153 media::ConvertYUVAToARGB( 154 video_frame->data(media::VideoFrame::kYPlane) + y_offset, 155 video_frame->data(media::VideoFrame::kUPlane) + uv_offset, 156 video_frame->data(media::VideoFrame::kVPlane) + uv_offset, 157 video_frame->data(media::VideoFrame::kAPlane), 158 static_cast<uint8*>(rgb_pixels), 159 video_frame->visible_rect().width(), 160 video_frame->visible_rect().height(), 161 video_frame->stride(media::VideoFrame::kYPlane), 162 video_frame->stride(media::VideoFrame::kUPlane), 163 video_frame->stride(media::VideoFrame::kAPlane), 164 row_bytes, 165 media::YV12); 166 break; 167 168 case media::VideoFrame::YV24: 169 libyuv::I444ToARGB( 170 video_frame->data(media::VideoFrame::kYPlane) + y_offset, 171 video_frame->stride(media::VideoFrame::kYPlane), 172 video_frame->data(media::VideoFrame::kUPlane) + uv_offset, 173 video_frame->stride(media::VideoFrame::kUPlane), 174 video_frame->data(media::VideoFrame::kVPlane) + uv_offset, 175 video_frame->stride(media::VideoFrame::kVPlane), 176 static_cast<uint8*>(rgb_pixels), 177 row_bytes, 178 video_frame->visible_rect().width(), 179 video_frame->visible_rect().height()); 180#if SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \ 181 SK_A32_SHIFT == 24 182 libyuv::ARGBToABGR(static_cast<uint8*>(rgb_pixels), 183 row_bytes, 184 static_cast<uint8*>(rgb_pixels), 185 row_bytes, 186 video_frame->visible_rect().width(), 187 video_frame->visible_rect().height()); 188#endif 189 break; 190 191 case media::VideoFrame::NATIVE_TEXTURE: { 192 DCHECK_EQ(video_frame->format(), media::VideoFrame::NATIVE_TEXTURE); 193 SkBitmap tmp; 194 tmp.installPixels( 195 SkImageInfo::MakeN32Premul(video_frame->visible_rect().width(), 196 video_frame->visible_rect().height()), 197 rgb_pixels, 198 row_bytes); 199 video_frame->ReadPixelsFromNativeTexture(tmp); 200 break; 201 } 202 default: 203 NOTREACHED(); 204 break; 205 } 206} 207 208// Generates an RGB image from a VideoFrame. 209class VideoImageGenerator : public SkImageGenerator { 210 public: 211 VideoImageGenerator(const scoped_refptr<VideoFrame>& frame) : frame_(frame) {} 212 virtual ~VideoImageGenerator() {} 213 214 void set_frame(const scoped_refptr<VideoFrame>& frame) { frame_ = frame; } 215 216 protected: 217 virtual bool onGetInfo(SkImageInfo* info) OVERRIDE { 218 info->fWidth = frame_->visible_rect().width(); 219 info->fHeight = frame_->visible_rect().height(); 220 info->fColorType = kN32_SkColorType; 221 info->fAlphaType = kPremul_SkAlphaType; 222 return true; 223 } 224 225 virtual bool onGetPixels(const SkImageInfo& info, 226 void* pixels, 227 size_t row_bytes, 228 SkPMColor ctable[], 229 int* ctable_count) OVERRIDE { 230 if (!frame_.get()) 231 return false; 232 if (!pixels) 233 return true; 234 // If skia couldn't do the YUV conversion, we will. 235 ConvertVideoFrameToRGBPixels(frame_, pixels, row_bytes); 236 frame_ = NULL; 237 return true; 238 } 239 240 virtual bool onGetYUV8Planes(SkISize sizes[3], 241 void* planes[3], 242 size_t row_bytes[3], 243 SkYUVColorSpace* color_space) OVERRIDE { 244 if (!frame_.get() || !IsYUV(frame_->format())) 245 return false; 246 247 if (color_space) { 248 if (IsJPEGColorSpace(frame_->format())) 249 *color_space = kJPEG_SkYUVColorSpace; 250 else 251 *color_space = kRec601_SkYUVColorSpace; 252 } 253 254 for (int plane = VideoFrame::kYPlane; plane <= VideoFrame::kVPlane; 255 ++plane) { 256 if (sizes) { 257 gfx::Size size; 258 size = 259 VideoFrame::PlaneSize(frame_->format(), 260 plane, 261 gfx::Size(frame_->visible_rect().width(), 262 frame_->visible_rect().height())); 263 sizes[plane].set(size.width(), size.height()); 264 } 265 if (row_bytes && planes) { 266 size_t offset; 267 int y_shift = (frame_->format() == media::VideoFrame::YV16) ? 0 : 1; 268 if (plane == media::VideoFrame::kYPlane) { 269 offset = (frame_->stride(media::VideoFrame::kYPlane) * 270 frame_->visible_rect().y()) + 271 frame_->visible_rect().x(); 272 } else { 273 offset = (frame_->stride(media::VideoFrame::kUPlane) * 274 (frame_->visible_rect().y() >> y_shift)) + 275 (frame_->visible_rect().x() >> 1); 276 } 277 row_bytes[plane] = static_cast<size_t>(frame_->stride(plane)); 278 planes[plane] = frame_->data(plane) + offset; 279 } 280 } 281 if (planes && row_bytes) 282 frame_ = NULL; 283 return true; 284 } 285 286 private: 287 scoped_refptr<VideoFrame> frame_; 288}; 289 290SkCanvasVideoRenderer::SkCanvasVideoRenderer() 291 : generator_(NULL), last_frame_timestamp_(media::kNoTimestamp()) { 292 last_frame_.setIsVolatile(true); 293} 294 295SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {} 296 297void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame, 298 SkCanvas* canvas, 299 const gfx::RectF& dest_rect, 300 uint8 alpha, 301 SkXfermode::Mode mode, 302 VideoRotation video_rotation) { 303 if (alpha == 0) { 304 return; 305 } 306 307 SkRect dest; 308 dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom()); 309 310 SkPaint paint; 311 paint.setAlpha(alpha); 312 313 // Paint black rectangle if there isn't a frame available or the 314 // frame has an unexpected format. 315 if (!video_frame.get() || !IsYUVOrNative(video_frame->format())) { 316 canvas->drawRect(dest, paint); 317 canvas->flush(); 318 return; 319 } 320 321 // Check if we should convert and update |last_frame_|. 322 if (last_frame_.isNull() || 323 video_frame->timestamp() != last_frame_timestamp_) { 324 generator_ = new VideoImageGenerator(video_frame); 325 326 // Note: This takes ownership of |generator_|. 327 if (!SkInstallDiscardablePixelRef(generator_, &last_frame_)) { 328 NOTREACHED(); 329 } 330 331 // TODO(rileya): Perform this rotation on the canvas, rather than allocating 332 // a new bitmap and copying. 333 switch (video_rotation) { 334 case VIDEO_ROTATION_0: 335 break; 336 case VIDEO_ROTATION_90: 337 last_frame_ = SkBitmapOperations::Rotate( 338 last_frame_, SkBitmapOperations::ROTATION_90_CW); 339 break; 340 case VIDEO_ROTATION_180: 341 last_frame_ = SkBitmapOperations::Rotate( 342 last_frame_, SkBitmapOperations::ROTATION_180_CW); 343 break; 344 case VIDEO_ROTATION_270: 345 last_frame_ = SkBitmapOperations::Rotate( 346 last_frame_, SkBitmapOperations::ROTATION_270_CW); 347 break; 348 } 349 350 // We copied the frame into a new bitmap and threw out the old one, so we 351 // no longer have a |generator_| around. This should be removed when the 352 // above TODO is addressed. 353 if (video_rotation != VIDEO_ROTATION_0) 354 generator_ = NULL; 355 356 last_frame_timestamp_ = video_frame->timestamp(); 357 } else if (generator_) { 358 generator_->set_frame(video_frame); 359 } 360 361 paint.setXfermodeMode(mode); 362 363 // Paint using |last_frame_|. 364 paint.setFilterLevel(SkPaint::kLow_FilterLevel); 365 canvas->drawBitmapRect(last_frame_, NULL, dest, &paint); 366 canvas->flush(); 367} 368 369void SkCanvasVideoRenderer::Copy(const scoped_refptr<VideoFrame>& video_frame, 370 SkCanvas* canvas) { 371 Paint(video_frame, 372 canvas, 373 video_frame->visible_rect(), 374 0xff, 375 SkXfermode::kSrc_Mode, 376 media::VIDEO_ROTATION_0); 377} 378 379} // namespace media 380