video_frame.cc revision 5c02ac1a9c1b504631c0a3d2b6e737b5d738bae1 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "media/base/video_frame.h"
6
7#include <algorithm>
8
9#include "base/bind.h"
10#include "base/callback_helpers.h"
11#include "base/logging.h"
12#include "base/memory/aligned_memory.h"
13#include "base/strings/string_piece.h"
14#include "gpu/command_buffer/common/mailbox_holder.h"
15#include "media/base/limits.h"
16#include "media/base/video_util.h"
17#include "third_party/skia/include/core/SkBitmap.h"
18
19namespace media {
20
21static inline size_t RoundUp(size_t value, size_t alignment) {
22  // Check that |alignment| is a power of 2.
23  DCHECK((alignment + (alignment - 1)) == (alignment | (alignment - 1)));
24  return ((value + (alignment - 1)) & ~(alignment - 1));
25}
26
27// static
28scoped_refptr<VideoFrame> VideoFrame::CreateFrame(
29    VideoFrame::Format format,
30    const gfx::Size& coded_size,
31    const gfx::Rect& visible_rect,
32    const gfx::Size& natural_size,
33    base::TimeDelta timestamp) {
34  // Since we're creating a new YUV frame (and allocating memory for it
35  // ourselves), we can pad the requested |coded_size| if necessary if the
36  // request does not line up on sample boundaries.
37  gfx::Size new_coded_size(coded_size);
38  switch (format) {
39    case VideoFrame::YV12:
40    case VideoFrame::YV12A:
41    case VideoFrame::I420:
42    case VideoFrame::YV12J:
43      new_coded_size.set_height((new_coded_size.height() + 1) / 2 * 2);
44    // Fallthrough.
45    case VideoFrame::YV16:
46      new_coded_size.set_width((new_coded_size.width() + 1) / 2 * 2);
47      break;
48    default:
49      LOG(FATAL) << "Only YUV formats supported: " << format;
50      return NULL;
51  }
52  DCHECK(IsValidConfig(format, new_coded_size, visible_rect, natural_size));
53  scoped_refptr<VideoFrame> frame(
54      new VideoFrame(format,
55                     new_coded_size,
56                     visible_rect,
57                     natural_size,
58                     scoped_ptr<gpu::MailboxHolder>(),
59                     timestamp,
60                     false));
61  frame->AllocateYUV();
62  return frame;
63}
64
65// static
66std::string VideoFrame::FormatToString(VideoFrame::Format format) {
67  switch (format) {
68    case VideoFrame::UNKNOWN:
69      return "UNKNOWN";
70    case VideoFrame::YV12:
71      return "YV12";
72    case VideoFrame::YV16:
73      return "YV16";
74    case VideoFrame::I420:
75      return "I420";
76    case VideoFrame::NATIVE_TEXTURE:
77      return "NATIVE_TEXTURE";
78#if defined(VIDEO_HOLE)
79    case VideoFrame::HOLE:
80      return "HOLE";
81#endif  // defined(VIDEO_HOLE)
82    case VideoFrame::YV12A:
83      return "YV12A";
84    case VideoFrame::YV12J:
85      return "YV12J";
86  }
87  NOTREACHED() << "Invalid videoframe format provided: " << format;
88  return "";
89}
90
91// static
92bool VideoFrame::IsValidConfig(VideoFrame::Format format,
93                               const gfx::Size& coded_size,
94                               const gfx::Rect& visible_rect,
95                               const gfx::Size& natural_size) {
96  // Check maximum limits for all formats.
97  if (coded_size.GetArea() > limits::kMaxCanvas ||
98      coded_size.width() > limits::kMaxDimension ||
99      coded_size.height() > limits::kMaxDimension ||
100      visible_rect.x() < 0 || visible_rect.y() < 0 ||
101      visible_rect.right() > coded_size.width() ||
102      visible_rect.bottom() > coded_size.height() ||
103      natural_size.GetArea() > limits::kMaxCanvas ||
104      natural_size.width() > limits::kMaxDimension ||
105      natural_size.height() > limits::kMaxDimension)
106    return false;
107
108  // Check format-specific width/height requirements.
109  switch (format) {
110    case VideoFrame::UNKNOWN:
111      return (coded_size.IsEmpty() && visible_rect.IsEmpty() &&
112              natural_size.IsEmpty());
113    case VideoFrame::YV12:
114    case VideoFrame::YV12J:
115    case VideoFrame::I420:
116    case VideoFrame::YV12A:
117      // YUV formats have width/height requirements due to chroma subsampling.
118      if (static_cast<size_t>(coded_size.height()) <
119          RoundUp(visible_rect.bottom(), 2))
120        return false;
121    // Fallthrough.
122    case VideoFrame::YV16:
123      if (static_cast<size_t>(coded_size.width()) <
124          RoundUp(visible_rect.right(), 2))
125        return false;
126      break;
127    case VideoFrame::NATIVE_TEXTURE:
128#if defined(VIDEO_HOLE)
129    case VideoFrame::HOLE:
130#endif  // defined(VIDEO_HOLE)
131      // NATIVE_TEXTURE and HOLE have no software-allocated buffers and are
132      // allowed to skip the below check and be empty.
133      return true;
134  }
135
136  // Check that software-allocated buffer formats are not empty.
137  return (!coded_size.IsEmpty() && !visible_rect.IsEmpty() &&
138          !natural_size.IsEmpty());
139}
140
141// static
142scoped_refptr<VideoFrame> VideoFrame::WrapNativeTexture(
143    scoped_ptr<gpu::MailboxHolder> mailbox_holder,
144    const ReleaseMailboxCB& mailbox_holder_release_cb,
145    const gfx::Size& coded_size,
146    const gfx::Rect& visible_rect,
147    const gfx::Size& natural_size,
148    base::TimeDelta timestamp,
149    const ReadPixelsCB& read_pixels_cb) {
150  scoped_refptr<VideoFrame> frame(new VideoFrame(NATIVE_TEXTURE,
151                                                 coded_size,
152                                                 visible_rect,
153                                                 natural_size,
154                                                 mailbox_holder.Pass(),
155                                                 timestamp,
156                                                 false));
157  frame->mailbox_holder_release_cb_ = mailbox_holder_release_cb;
158  frame->read_pixels_cb_ = read_pixels_cb;
159
160  return frame;
161}
162
163void VideoFrame::ReadPixelsFromNativeTexture(const SkBitmap& pixels) {
164  DCHECK_EQ(format_, NATIVE_TEXTURE);
165  if (!read_pixels_cb_.is_null())
166    read_pixels_cb_.Run(pixels);
167}
168
169// static
170scoped_refptr<VideoFrame> VideoFrame::WrapExternalPackedMemory(
171    Format format,
172    const gfx::Size& coded_size,
173    const gfx::Rect& visible_rect,
174    const gfx::Size& natural_size,
175    uint8* data,
176    size_t data_size,
177    base::SharedMemoryHandle handle,
178    base::TimeDelta timestamp,
179    const base::Closure& no_longer_needed_cb) {
180  if (!IsValidConfig(format, coded_size, visible_rect, natural_size))
181    return NULL;
182  if (data_size < AllocationSize(format, coded_size))
183    return NULL;
184
185  switch (format) {
186    case I420: {
187      scoped_refptr<VideoFrame> frame(
188          new VideoFrame(format,
189                         coded_size,
190                         visible_rect,
191                         natural_size,
192                         scoped_ptr<gpu::MailboxHolder>(),
193                         timestamp,
194                         false));
195      frame->shared_memory_handle_ = handle;
196      frame->strides_[kYPlane] = coded_size.width();
197      frame->strides_[kUPlane] = coded_size.width() / 2;
198      frame->strides_[kVPlane] = coded_size.width() / 2;
199      frame->data_[kYPlane] = data;
200      frame->data_[kUPlane] = data + coded_size.GetArea();
201      frame->data_[kVPlane] = data + (coded_size.GetArea() * 5 / 4);
202      frame->no_longer_needed_cb_ = no_longer_needed_cb;
203      return frame;
204    }
205    default:
206      NOTIMPLEMENTED();
207      return NULL;
208  }
209}
210
211// static
212scoped_refptr<VideoFrame> VideoFrame::WrapExternalYuvData(
213    Format format,
214    const gfx::Size& coded_size,
215    const gfx::Rect& visible_rect,
216    const gfx::Size& natural_size,
217    int32 y_stride,
218    int32 u_stride,
219    int32 v_stride,
220    uint8* y_data,
221    uint8* u_data,
222    uint8* v_data,
223    base::TimeDelta timestamp,
224    const base::Closure& no_longer_needed_cb) {
225  if (!IsValidConfig(format, coded_size, visible_rect, natural_size))
226    return NULL;
227
228  scoped_refptr<VideoFrame> frame(
229      new VideoFrame(format,
230                     coded_size,
231                     visible_rect,
232                     natural_size,
233                     scoped_ptr<gpu::MailboxHolder>(),
234                     timestamp,
235                     false));
236  frame->strides_[kYPlane] = y_stride;
237  frame->strides_[kUPlane] = u_stride;
238  frame->strides_[kVPlane] = v_stride;
239  frame->data_[kYPlane] = y_data;
240  frame->data_[kUPlane] = u_data;
241  frame->data_[kVPlane] = v_data;
242  frame->no_longer_needed_cb_ = no_longer_needed_cb;
243  return frame;
244}
245
246// static
247scoped_refptr<VideoFrame> VideoFrame::WrapVideoFrame(
248      const scoped_refptr<VideoFrame>& frame,
249      const gfx::Rect& visible_rect,
250      const gfx::Size& natural_size,
251      const base::Closure& no_longer_needed_cb) {
252  // NATIVE_TEXTURE frames need mailbox info propagated, and there's no support
253  // for that here yet, see http://crbug/362521.
254  CHECK(frame->format() != NATIVE_TEXTURE);
255
256  DCHECK(frame->visible_rect().Contains(visible_rect));
257  scoped_refptr<VideoFrame> wrapped_frame(
258      new VideoFrame(frame->format(),
259                     frame->coded_size(),
260                     visible_rect,
261                     natural_size,
262                     scoped_ptr<gpu::MailboxHolder>(),
263                     frame->timestamp(),
264                     frame->end_of_stream()));
265
266  for (size_t i = 0; i < NumPlanes(frame->format()); ++i) {
267    wrapped_frame->strides_[i] = frame->stride(i);
268    wrapped_frame->data_[i] = frame->data(i);
269  }
270
271  wrapped_frame->no_longer_needed_cb_ = no_longer_needed_cb;
272  return wrapped_frame;
273}
274
275// static
276scoped_refptr<VideoFrame> VideoFrame::CreateEOSFrame() {
277  return new VideoFrame(VideoFrame::UNKNOWN,
278                        gfx::Size(),
279                        gfx::Rect(),
280                        gfx::Size(),
281                        scoped_ptr<gpu::MailboxHolder>(),
282                        kNoTimestamp(),
283                        true);
284}
285
286// static
287scoped_refptr<VideoFrame> VideoFrame::CreateColorFrame(
288    const gfx::Size& size,
289    uint8 y, uint8 u, uint8 v,
290    base::TimeDelta timestamp) {
291  scoped_refptr<VideoFrame> frame = VideoFrame::CreateFrame(
292      VideoFrame::YV12, size, gfx::Rect(size), size, timestamp);
293  FillYUV(frame.get(), y, u, v);
294  return frame;
295}
296
297// static
298scoped_refptr<VideoFrame> VideoFrame::CreateBlackFrame(const gfx::Size& size) {
299  const uint8 kBlackY = 0x00;
300  const uint8 kBlackUV = 0x80;
301  const base::TimeDelta kZero;
302  return CreateColorFrame(size, kBlackY, kBlackUV, kBlackUV, kZero);
303}
304
305#if defined(VIDEO_HOLE)
306// This block and other blocks wrapped around #if defined(VIDEO_HOLE) is not
307// maintained by the general compositor team. Please contact the following
308// people instead:
309//
310// wonsik@chromium.org
311// ycheo@chromium.org
312
313// static
314scoped_refptr<VideoFrame> VideoFrame::CreateHoleFrame(
315    const gfx::Size& size) {
316  DCHECK(IsValidConfig(VideoFrame::HOLE, size, gfx::Rect(size), size));
317  scoped_refptr<VideoFrame> frame(
318      new VideoFrame(VideoFrame::HOLE,
319                     size,
320                     gfx::Rect(size),
321                     size,
322                     scoped_ptr<gpu::MailboxHolder>(),
323                     base::TimeDelta(),
324                     false));
325  return frame;
326}
327#endif  // defined(VIDEO_HOLE)
328
329// static
330size_t VideoFrame::NumPlanes(Format format) {
331  switch (format) {
332    case VideoFrame::NATIVE_TEXTURE:
333#if defined(VIDEO_HOLE)
334    case VideoFrame::HOLE:
335#endif  // defined(VIDEO_HOLE)
336      return 0;
337    case VideoFrame::YV12:
338    case VideoFrame::YV16:
339    case VideoFrame::I420:
340    case VideoFrame::YV12J:
341      return 3;
342    case VideoFrame::YV12A:
343      return 4;
344    case VideoFrame::UNKNOWN:
345      break;
346  }
347  NOTREACHED() << "Unsupported video frame format: " << format;
348  return 0;
349}
350
351
352// static
353size_t VideoFrame::AllocationSize(Format format, const gfx::Size& coded_size) {
354  size_t total = 0;
355  for (size_t i = 0; i < NumPlanes(format); ++i)
356    total += PlaneAllocationSize(format, i, coded_size);
357  return total;
358}
359
360// static
361gfx::Size VideoFrame::PlaneSize(Format format,
362                                size_t plane,
363                                const gfx::Size& coded_size) {
364  const int width = RoundUp(coded_size.width(), 2);
365  const int height = RoundUp(coded_size.height(), 2);
366  switch (format) {
367    case VideoFrame::YV12:
368    case VideoFrame::YV12J:
369    case VideoFrame::I420: {
370      switch (plane) {
371        case VideoFrame::kYPlane:
372          return gfx::Size(width, height);
373        case VideoFrame::kUPlane:
374        case VideoFrame::kVPlane:
375          return gfx::Size(width / 2, height / 2);
376        default:
377          break;
378      }
379    }
380    case VideoFrame::YV12A: {
381      switch (plane) {
382        case VideoFrame::kYPlane:
383        case VideoFrame::kAPlane:
384          return gfx::Size(width, height);
385        case VideoFrame::kUPlane:
386        case VideoFrame::kVPlane:
387          return gfx::Size(width / 2, height / 2);
388        default:
389          break;
390      }
391    }
392    case VideoFrame::YV16: {
393      switch (plane) {
394        case VideoFrame::kYPlane:
395          return gfx::Size(width, height);
396        case VideoFrame::kUPlane:
397        case VideoFrame::kVPlane:
398          return gfx::Size(width / 2, height);
399        default:
400          break;
401      }
402    }
403    case VideoFrame::UNKNOWN:
404    case VideoFrame::NATIVE_TEXTURE:
405#if defined(VIDEO_HOLE)
406    case VideoFrame::HOLE:
407#endif  // defined(VIDEO_HOLE)
408      break;
409  }
410  NOTREACHED() << "Unsupported video frame format/plane: "
411               << format << "/" << plane;
412  return gfx::Size();
413}
414
415size_t VideoFrame::PlaneAllocationSize(Format format,
416                                       size_t plane,
417                                       const gfx::Size& coded_size) {
418  // VideoFrame formats are (so far) all YUV and 1 byte per sample.
419  return PlaneSize(format, plane, coded_size).GetArea();
420}
421
422// Release data allocated by AllocateYUV().
423static void ReleaseData(uint8* data) {
424  DCHECK(data);
425  base::AlignedFree(data);
426}
427
428void VideoFrame::AllocateYUV() {
429  DCHECK(format_ == VideoFrame::YV12 || format_ == VideoFrame::YV16 ||
430         format_ == VideoFrame::YV12A || format_ == VideoFrame::I420 ||
431         format_ == VideoFrame::YV12J);
432  // Align Y rows at least at 16 byte boundaries.  The stride for both
433  // YV12 and YV16 is 1/2 of the stride of Y.  For YV12, every row of bytes for
434  // U and V applies to two rows of Y (one byte of UV for 4 bytes of Y), so in
435  // the case of YV12 the strides are identical for the same width surface, but
436  // the number of bytes allocated for YV12 is 1/2 the amount for U & V as
437  // YV16. We also round the height of the surface allocated to be an even
438  // number to avoid any potential of faulting by code that attempts to access
439  // the Y values of the final row, but assumes that the last row of U & V
440  // applies to a full two rows of Y. YV12A is the same as YV12, but with an
441  // additional alpha plane that has the same size and alignment as the Y plane.
442
443  size_t y_stride = RoundUp(row_bytes(VideoFrame::kYPlane),
444                            kFrameSizeAlignment);
445  size_t uv_stride = RoundUp(row_bytes(VideoFrame::kUPlane),
446                             kFrameSizeAlignment);
447  // The *2 here is because some formats (e.g. h264) allow interlaced coding,
448  // and then the size needs to be a multiple of two macroblocks (vertically).
449  // See libavcodec/utils.c:avcodec_align_dimensions2().
450  size_t y_height = RoundUp(coded_size_.height(), kFrameSizeAlignment * 2);
451  size_t uv_height =
452      (format_ == VideoFrame::YV12 || format_ == VideoFrame::YV12A ||
453       format_ == VideoFrame::I420)
454          ? y_height / 2
455          : y_height;
456  size_t y_bytes = y_height * y_stride;
457  size_t uv_bytes = uv_height * uv_stride;
458  size_t a_bytes = format_ == VideoFrame::YV12A ? y_bytes : 0;
459
460  // The extra line of UV being allocated is because h264 chroma MC
461  // overreads by one line in some cases, see libavcodec/utils.c:
462  // avcodec_align_dimensions2() and libavcodec/x86/h264_chromamc.asm:
463  // put_h264_chroma_mc4_ssse3().
464  uint8* data = reinterpret_cast<uint8*>(
465      base::AlignedAlloc(
466          y_bytes + (uv_bytes * 2 + uv_stride) + a_bytes + kFrameSizePadding,
467          kFrameAddressAlignment));
468  no_longer_needed_cb_ = base::Bind(&ReleaseData, data);
469  COMPILE_ASSERT(0 == VideoFrame::kYPlane, y_plane_data_must_be_index_0);
470  data_[VideoFrame::kYPlane] = data;
471  data_[VideoFrame::kUPlane] = data + y_bytes;
472  data_[VideoFrame::kVPlane] = data + y_bytes + uv_bytes;
473  strides_[VideoFrame::kYPlane] = y_stride;
474  strides_[VideoFrame::kUPlane] = uv_stride;
475  strides_[VideoFrame::kVPlane] = uv_stride;
476  if (format_ == YV12A) {
477    data_[VideoFrame::kAPlane] = data + y_bytes + (2 * uv_bytes);
478    strides_[VideoFrame::kAPlane] = y_stride;
479  }
480}
481
482VideoFrame::VideoFrame(VideoFrame::Format format,
483                       const gfx::Size& coded_size,
484                       const gfx::Rect& visible_rect,
485                       const gfx::Size& natural_size,
486                       scoped_ptr<gpu::MailboxHolder> mailbox_holder,
487                       base::TimeDelta timestamp,
488                       bool end_of_stream)
489    : format_(format),
490      coded_size_(coded_size),
491      visible_rect_(visible_rect),
492      natural_size_(natural_size),
493      mailbox_holder_(mailbox_holder.Pass()),
494      shared_memory_handle_(base::SharedMemory::NULLHandle()),
495      timestamp_(timestamp),
496      end_of_stream_(end_of_stream) {
497  DCHECK(IsValidConfig(format_, coded_size_, visible_rect_, natural_size_));
498
499  memset(&strides_, 0, sizeof(strides_));
500  memset(&data_, 0, sizeof(data_));
501}
502
503VideoFrame::~VideoFrame() {
504  if (!mailbox_holder_release_cb_.is_null()) {
505    std::vector<uint32> release_sync_points;
506    {
507      base::AutoLock locker(release_sync_point_lock_);
508      release_sync_points_.swap(release_sync_points);
509    }
510    base::ResetAndReturn(&mailbox_holder_release_cb_).Run(release_sync_points);
511  }
512  if (!no_longer_needed_cb_.is_null())
513    base::ResetAndReturn(&no_longer_needed_cb_).Run();
514}
515
516bool VideoFrame::IsValidPlane(size_t plane) const {
517  return (plane < NumPlanes(format_));
518}
519
520int VideoFrame::stride(size_t plane) const {
521  DCHECK(IsValidPlane(plane));
522  return strides_[plane];
523}
524
525int VideoFrame::row_bytes(size_t plane) const {
526  DCHECK(IsValidPlane(plane));
527  int width = coded_size_.width();
528  switch (format_) {
529    // Planar, 8bpp.
530    case YV12A:
531      if (plane == kAPlane)
532        return width;
533    // Fallthrough.
534    case YV12:
535    case YV16:
536    case I420:
537    case YV12J:
538      if (plane == kYPlane)
539        return width;
540      return RoundUp(width, 2) / 2;
541
542    default:
543      break;
544  }
545
546  // Intentionally leave out non-production formats.
547  NOTREACHED() << "Unsupported video frame format: " << format_;
548  return 0;
549}
550
551int VideoFrame::rows(size_t plane) const {
552  DCHECK(IsValidPlane(plane));
553  int height = coded_size_.height();
554  switch (format_) {
555    case YV16:
556      return height;
557
558    case YV12A:
559      if (plane == kAPlane)
560        return height;
561    // Fallthrough.
562    case YV12:
563    case I420:
564      if (plane == kYPlane)
565        return height;
566      return RoundUp(height, 2) / 2;
567
568    default:
569      break;
570  }
571
572  // Intentionally leave out non-production formats.
573  NOTREACHED() << "Unsupported video frame format: " << format_;
574  return 0;
575}
576
577uint8* VideoFrame::data(size_t plane) const {
578  DCHECK(IsValidPlane(plane));
579  return data_[plane];
580}
581
582const gpu::MailboxHolder* VideoFrame::mailbox_holder() const {
583  DCHECK_EQ(format_, NATIVE_TEXTURE);
584  return mailbox_holder_.get();
585}
586
587base::SharedMemoryHandle VideoFrame::shared_memory_handle() const {
588  return shared_memory_handle_;
589}
590
591void VideoFrame::AppendReleaseSyncPoint(uint32 sync_point) {
592  DCHECK_EQ(format_, NATIVE_TEXTURE);
593  if (!sync_point)
594    return;
595  base::AutoLock locker(release_sync_point_lock_);
596  release_sync_points_.push_back(sync_point);
597}
598
599void VideoFrame::HashFrameForTesting(base::MD5Context* context) {
600  for (int plane = 0; plane < kMaxPlanes; ++plane) {
601    if (!IsValidPlane(plane))
602      break;
603    for (int row = 0; row < rows(plane); ++row) {
604      base::MD5Update(context, base::StringPiece(
605          reinterpret_cast<char*>(data(plane) + stride(plane) * row),
606          row_bytes(plane)));
607    }
608  }
609}
610
611}  // namespace media
612