videoframe.cc revision bdcf38c89446b1b464a646414f6cd7573a190bd1 
1/*
2 * libjingle
3 * Copyright 2011 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 *  1. Redistributions of source code must retain the above copyright notice,
9 *     this list of conditions and the following disclaimer.
10 *  2. Redistributions in binary form must reproduce the above copyright notice,
11 *     this list of conditions and the following disclaimer in the documentation
12 *     and/or other materials provided with the distribution.
13 *  3. The name of the author may not be used to endorse or promote products
14 *     derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#include "talk/media/base/videoframe.h"
29
30#include <string.h>
31
32#if !defined(DISABLE_YUV)
33#include "libyuv/compare.h"
34#include "libyuv/planar_functions.h"
35#include "libyuv/scale.h"
36#endif
37
38#include "talk/media/base/videocommon.h"
39#include "webrtc/base/logging.h"
40
41namespace cricket {
42
43// Round to 2 pixels because Chroma channels are half size.
44#define ROUNDTO2(v) (v & ~1)
45
46rtc::StreamResult VideoFrame::Write(rtc::StreamInterface* stream,
47                                          int* error) {
48  rtc::StreamResult result = rtc::SR_SUCCESS;
49  const uint8* src_y = GetYPlane();
50  const uint8* src_u = GetUPlane();
51  const uint8* src_v = GetVPlane();
52  if (!src_y || !src_u || !src_v) {
53    return result;  // Nothing to write.
54  }
55  const int32 y_pitch = GetYPitch();
56  const int32 u_pitch = GetUPitch();
57  const int32 v_pitch = GetVPitch();
58  const size_t width = GetWidth();
59  const size_t height = GetHeight();
60  const size_t half_width = (width + 1) >> 1;
61  const size_t half_height = (height + 1) >> 1;
62  // Write Y.
63  for (size_t row = 0; row < height; ++row) {
64    result = stream->Write(src_y + row * y_pitch, width, NULL, error);
65    if (result != rtc::SR_SUCCESS) {
66      return result;
67    }
68  }
69  // Write U.
70  for (size_t row = 0; row < half_height; ++row) {
71    result = stream->Write(src_u + row * u_pitch, half_width, NULL, error);
72    if (result != rtc::SR_SUCCESS) {
73      return result;
74    }
75  }
76  // Write V.
77  for (size_t row = 0; row < half_height; ++row) {
78    result = stream->Write(src_v + row * v_pitch, half_width, NULL, error);
79    if (result != rtc::SR_SUCCESS) {
80      return result;
81    }
82  }
83  return result;
84}
85
86bool VideoFrame::CopyToPlanes(
87    uint8* dst_y, uint8* dst_u, uint8* dst_v,
88    int32 dst_pitch_y, int32 dst_pitch_u, int32 dst_pitch_v) const {
89#if !defined(DISABLE_YUV)
90  int32 src_width = static_cast<int>(GetWidth());
91  int32 src_height = static_cast<int>(GetHeight());
92  return libyuv::I420Copy(GetYPlane(), GetYPitch(),
93                          GetUPlane(), GetUPitch(),
94                          GetVPlane(), GetVPitch(),
95                          dst_y, dst_pitch_y,
96                          dst_u, dst_pitch_u,
97                          dst_v, dst_pitch_v,
98                          src_width, src_height) == 0;
99#else
100  int uv_size = GetUPitch() * GetChromaHeight();
101  memcpy(dst_y, GetYPlane(), GetWidth() * GetHeight());
102  memcpy(dst_u, GetUPlane(), uv_size);
103  memcpy(dst_v, GetVPlane(), uv_size);
104  return true;
105#endif
106}
107
108void VideoFrame::CopyToFrame(VideoFrame* dst) const {
109  if (!dst) {
110    LOG(LS_ERROR) << "NULL dst pointer.";
111    return;
112  }
113
114  CopyToPlanes(dst->GetYPlane(), dst->GetUPlane(), dst->GetVPlane(),
115               dst->GetYPitch(), dst->GetUPitch(), dst->GetVPitch());
116}
117
118size_t VideoFrame::ConvertToRgbBuffer(uint32 to_fourcc,
119                                      uint8* buffer,
120                                      size_t size,
121                                      int stride_rgb) const {
122  const size_t needed = std::abs(stride_rgb) * GetHeight();
123  if (size < needed) {
124    LOG(LS_WARNING) << "RGB buffer is not large enough";
125    return needed;
126  }
127
128  if (libyuv::ConvertFromI420(GetYPlane(), GetYPitch(), GetUPlane(),
129                              GetUPitch(), GetVPlane(), GetVPitch(), buffer,
130                              stride_rgb, static_cast<int>(GetWidth()),
131                              static_cast<int>(GetHeight()), to_fourcc)) {
132    LOG(LS_ERROR) << "RGB type not supported: " << to_fourcc;
133    return 0;  // 0 indicates error
134  }
135  return needed;
136}
137
138// TODO(fbarchard): Handle odd width/height with rounding.
139void VideoFrame::StretchToPlanes(
140    uint8* dst_y, uint8* dst_u, uint8* dst_v,
141    int32 dst_pitch_y, int32 dst_pitch_u, int32 dst_pitch_v,
142    size_t width, size_t height, bool interpolate, bool vert_crop) const {
143  if (!GetYPlane() || !GetUPlane() || !GetVPlane()) {
144    LOG(LS_ERROR) << "NULL plane pointer.";
145    return;
146  }
147
148  size_t src_width = GetWidth();
149  size_t src_height = GetHeight();
150  if (width == src_width && height == src_height) {
151    CopyToPlanes(dst_y, dst_u, dst_v, dst_pitch_y, dst_pitch_u, dst_pitch_v);
152    return;
153  }
154  const uint8* src_y = GetYPlane();
155  const uint8* src_u = GetUPlane();
156  const uint8* src_v = GetVPlane();
157
158  if (vert_crop) {
159    // Adjust the input width:height ratio to be the same as the output ratio.
160    if (src_width * height > src_height * width) {
161      // Reduce the input width, but keep size/position aligned for YuvScaler
162      src_width = ROUNDTO2(src_height * width / height);
163      int32 iwidth_offset = ROUNDTO2((GetWidth() - src_width) / 2);
164      src_y += iwidth_offset;
165      src_u += iwidth_offset / 2;
166      src_v += iwidth_offset / 2;
167    } else if (src_width * height < src_height * width) {
168      // Reduce the input height.
169      src_height = src_width * height / width;
170      int32 iheight_offset = static_cast<int32>(
171          (GetHeight() - src_height) >> 2);
172      iheight_offset <<= 1;  // Ensure that iheight_offset is even.
173      src_y += iheight_offset * GetYPitch();
174      src_u += iheight_offset / 2 * GetUPitch();
175      src_v += iheight_offset / 2 * GetVPitch();
176    }
177  }
178
179  // TODO(fbarchard): Implement a simple scale for non-libyuv.
180#if !defined(DISABLE_YUV)
181  // Scale to the output I420 frame.
182  libyuv::Scale(src_y, src_u, src_v,
183                GetYPitch(), GetUPitch(), GetVPitch(),
184                static_cast<int>(src_width), static_cast<int>(src_height),
185                dst_y, dst_u, dst_v, dst_pitch_y, dst_pitch_u, dst_pitch_v,
186                static_cast<int>(width), static_cast<int>(height), interpolate);
187#endif
188}
189
190size_t VideoFrame::StretchToBuffer(size_t dst_width, size_t dst_height,
191                                   uint8* dst_buffer, size_t size,
192                                   bool interpolate, bool vert_crop) const {
193  if (!dst_buffer) {
194    LOG(LS_ERROR) << "NULL dst_buffer pointer.";
195    return 0;
196  }
197
198  size_t needed = SizeOf(dst_width, dst_height);
199  if (needed <= size) {
200    uint8* dst_y = dst_buffer;
201    uint8* dst_u = dst_y + dst_width * dst_height;
202    uint8* dst_v = dst_u + ((dst_width + 1) >> 1) * ((dst_height + 1) >> 1);
203    StretchToPlanes(dst_y, dst_u, dst_v,
204                    static_cast<int32>(dst_width),
205                    static_cast<int32>((dst_width + 1) >> 1),
206                    static_cast<int32>((dst_width + 1) >> 1),
207                    dst_width, dst_height, interpolate, vert_crop);
208  }
209  return needed;
210}
211
212void VideoFrame::StretchToFrame(VideoFrame* dst,
213                                bool interpolate, bool vert_crop) const {
214  if (!dst) {
215    LOG(LS_ERROR) << "NULL dst pointer.";
216    return;
217  }
218
219  StretchToPlanes(dst->GetYPlane(), dst->GetUPlane(), dst->GetVPlane(),
220                  dst->GetYPitch(), dst->GetUPitch(), dst->GetVPitch(),
221                  dst->GetWidth(), dst->GetHeight(),
222                  interpolate, vert_crop);
223  dst->SetElapsedTime(GetElapsedTime());
224  dst->SetTimeStamp(GetTimeStamp());
225}
226
227VideoFrame* VideoFrame::Stretch(size_t dst_width, size_t dst_height,
228                                bool interpolate, bool vert_crop) const {
229  VideoFrame* dest = CreateEmptyFrame(static_cast<int>(dst_width),
230                                      static_cast<int>(dst_height),
231                                      GetPixelWidth(), GetPixelHeight(),
232                                      GetElapsedTime(), GetTimeStamp());
233  if (dest) {
234    StretchToFrame(dest, interpolate, vert_crop);
235  }
236  return dest;
237}
238
239bool VideoFrame::SetToBlack() {
240#if !defined(DISABLE_YUV)
241  return libyuv::I420Rect(GetYPlane(), GetYPitch(),
242                          GetUPlane(), GetUPitch(),
243                          GetVPlane(), GetVPitch(),
244                          0, 0,
245                          static_cast<int>(GetWidth()),
246                          static_cast<int>(GetHeight()),
247                          16, 128, 128) == 0;
248#else
249  int uv_size = GetUPitch() * GetChromaHeight();
250  memset(GetYPlane(), 16, GetWidth() * GetHeight());
251  memset(GetUPlane(), 128, uv_size);
252  memset(GetVPlane(), 128, uv_size);
253  return true;
254#endif
255}
256
257static const size_t kMaxSampleSize = 1000000000u;
258// Returns whether a sample is valid.
259bool VideoFrame::Validate(uint32 fourcc, int w, int h,
260                          const uint8 *sample, size_t sample_size) {
261  if (h < 0) {
262    h = -h;
263  }
264  // 16384 is maximum resolution for VP8 codec.
265  if (w < 1 || w > 16384 || h < 1 || h > 16384) {
266    LOG(LS_ERROR) << "Invalid dimensions: " << w << "x" << h;
267    return false;
268  }
269  uint32 format = CanonicalFourCC(fourcc);
270  int expected_bpp = 8;
271  switch (format) {
272    case FOURCC_I400:
273    case FOURCC_RGGB:
274    case FOURCC_BGGR:
275    case FOURCC_GRBG:
276    case FOURCC_GBRG:
277      expected_bpp = 8;
278      break;
279    case FOURCC_I420:
280    case FOURCC_I411:
281    case FOURCC_YU12:
282    case FOURCC_YV12:
283    case FOURCC_M420:
284    case FOURCC_Q420:
285    case FOURCC_NV21:
286    case FOURCC_NV12:
287      expected_bpp = 12;
288      break;
289    case FOURCC_I422:
290    case FOURCC_YV16:
291    case FOURCC_YUY2:
292    case FOURCC_UYVY:
293    case FOURCC_RGBP:
294    case FOURCC_RGBO:
295    case FOURCC_R444:
296      expected_bpp = 16;
297      break;
298    case FOURCC_I444:
299    case FOURCC_YV24:
300    case FOURCC_24BG:
301    case FOURCC_RAW:
302      expected_bpp = 24;
303      break;
304
305    case FOURCC_ABGR:
306    case FOURCC_BGRA:
307    case FOURCC_ARGB:
308      expected_bpp = 32;
309      break;
310
311    case FOURCC_MJPG:
312    case FOURCC_H264:
313      expected_bpp = 0;
314      break;
315    default:
316      expected_bpp = 8;  // Expect format is at least 8 bits per pixel.
317      break;
318  }
319  size_t expected_size = (w * expected_bpp + 7) / 8 * h;
320  // For compressed formats, expect 4 bits per 16 x 16 macro.  I420 would be
321  // 6 bits, but grey can be 4 bits.
322  if (expected_bpp == 0) {
323    expected_size = ((w + 15) / 16) * ((h + 15) / 16) * 4 / 8;
324  }
325  if (sample == NULL) {
326    LOG(LS_ERROR) << "NULL sample pointer."
327                  << " format: " << GetFourccName(format)
328                  << " bpp: " << expected_bpp
329                  << " size: " << w << "x" << h
330                  << " expected: " << expected_size
331                  << " " << sample_size;
332    return false;
333  }
334  // TODO(fbarchard): Make function to dump information about frames.
335  uint8 four_samples[4] = { 0, 0, 0, 0 };
336  for (size_t i = 0; i < ARRAY_SIZE(four_samples) && i < sample_size; ++i) {
337    four_samples[i] = sample[i];
338  }
339  if (sample_size < expected_size) {
340    LOG(LS_ERROR) << "Size field is too small."
341                  << " format: " << GetFourccName(format)
342                  << " bpp: " << expected_bpp
343                  << " size: " << w << "x" << h
344                  << " " << sample_size
345                  << " expected: " << expected_size
346                  << " sample[0..3]: " << static_cast<int>(four_samples[0])
347                  << ", " << static_cast<int>(four_samples[1])
348                  << ", " << static_cast<int>(four_samples[2])
349                  << ", " << static_cast<int>(four_samples[3]);
350    return false;
351  }
352  if (sample_size > kMaxSampleSize) {
353    LOG(LS_WARNING) << "Size field is invalid."
354                    << " format: " << GetFourccName(format)
355                    << " bpp: " << expected_bpp
356                    << " size: " << w << "x" << h
357                    << " " << sample_size
358                    << " expected: " << 2 * expected_size
359                    << " sample[0..3]: " << static_cast<int>(four_samples[0])
360                    << ", " << static_cast<int>(four_samples[1])
361                    << ", " << static_cast<int>(four_samples[2])
362                    << ", " << static_cast<int>(four_samples[3]);
363    return false;
364  }
365  // Show large size warning once every 100 frames.
366  // TODO(fbarchard): Make frame counter atomic for thread safety.
367  static int large_warn100 = 0;
368  size_t large_expected_size = expected_size * 2;
369  if (expected_bpp >= 8 &&
370      (sample_size > large_expected_size || sample_size > kMaxSampleSize) &&
371      large_warn100 % 100 == 0) {
372    ++large_warn100;
373    LOG(LS_WARNING) << "Size field is too large."
374                    << " format: " << GetFourccName(format)
375                    << " bpp: " << expected_bpp
376                    << " size: " << w << "x" << h
377                    << " bytes: " << sample_size
378                    << " expected: " << large_expected_size
379                    << " sample[0..3]: " << static_cast<int>(four_samples[0])
380                    << ", " << static_cast<int>(four_samples[1])
381                    << ", " << static_cast<int>(four_samples[2])
382                    << ", " << static_cast<int>(four_samples[3]);
383  }
384
385  // TODO(fbarchard): Add duplicate pixel check.
386  // TODO(fbarchard): Use frame counter atomic for thread safety.
387  static bool valid_once = true;
388  if (valid_once) {
389    valid_once = false;
390    LOG(LS_INFO) << "Validate frame passed."
391                 << " format: " << GetFourccName(format)
392                 << " bpp: " << expected_bpp
393                 << " size: " << w << "x" << h
394                 << " bytes: " << sample_size
395                 << " expected: " << expected_size
396                 << " sample[0..3]: " << static_cast<int>(four_samples[0])
397                 << ", " << static_cast<int>(four_samples[1])
398                 << ", " << static_cast<int>(four_samples[2])
399                 << ", " << static_cast<int>(four_samples[3]);
400  }
401  return true;
402}
403
404}  // namespace cricket
405