ffmpeg_video_decoder.cc revision cedac228d2dd51db4b79ea1e72c7f249408ee061 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "media/filters/ffmpeg_video_decoder.h"
6
7#include <algorithm>
8#include <string>
9
10#include "base/bind.h"
11#include "base/callback_helpers.h"
12#include "base/command_line.h"
13#include "base/location.h"
14#include "base/single_thread_task_runner.h"
15#include "base/strings/string_number_conversions.h"
16#include "media/base/bind_to_current_loop.h"
17#include "media/base/decoder_buffer.h"
18#include "media/base/limits.h"
19#include "media/base/media_switches.h"
20#include "media/base/pipeline.h"
21#include "media/base/video_decoder_config.h"
22#include "media/base/video_frame.h"
23#include "media/base/video_util.h"
24#include "media/ffmpeg/ffmpeg_common.h"
25#include "media/filters/ffmpeg_glue.h"
26
27namespace media {
28
29// Always try to use three threads for video decoding.  There is little reason
30// not to since current day CPUs tend to be multi-core and we measured
31// performance benefits on older machines such as P4s with hyperthreading.
32//
33// Handling decoding on separate threads also frees up the pipeline thread to
34// continue processing. Although it'd be nice to have the option of a single
35// decoding thread, FFmpeg treats having one thread the same as having zero
36// threads (i.e., avcodec_decode_video() will execute on the calling thread).
37// Yet another reason for having two threads :)
38static const int kDecodeThreads = 2;
39static const int kMaxDecodeThreads = 16;
40
41// Returns the number of threads given the FFmpeg CodecID. Also inspects the
42// command line for a valid --video-threads flag.
43static int GetThreadCount(AVCodecID codec_id) {
44  // Refer to http://crbug.com/93932 for tsan suppressions on decoding.
45  int decode_threads = kDecodeThreads;
46
47  const CommandLine* cmd_line = CommandLine::ForCurrentProcess();
48  std::string threads(cmd_line->GetSwitchValueASCII(switches::kVideoThreads));
49  if (threads.empty() || !base::StringToInt(threads, &decode_threads))
50    return decode_threads;
51
52  decode_threads = std::max(decode_threads, 0);
53  decode_threads = std::min(decode_threads, kMaxDecodeThreads);
54  return decode_threads;
55}
56
57static int GetVideoBufferImpl(struct AVCodecContext* s,
58                              AVFrame* frame,
59                              int flags) {
60  FFmpegVideoDecoder* decoder = static_cast<FFmpegVideoDecoder*>(s->opaque);
61  return decoder->GetVideoBuffer(s, frame, flags);
62}
63
64static void ReleaseVideoBufferImpl(void* opaque, uint8* data) {
65  scoped_refptr<VideoFrame> video_frame;
66  video_frame.swap(reinterpret_cast<VideoFrame**>(&opaque));
67}
68
69FFmpegVideoDecoder::FFmpegVideoDecoder(
70    const scoped_refptr<base::SingleThreadTaskRunner>& task_runner)
71    : task_runner_(task_runner), state_(kUninitialized) {}
72
73int FFmpegVideoDecoder::GetVideoBuffer(struct AVCodecContext* codec_context,
74                                       AVFrame* frame,
75                                       int flags) {
76  // Don't use |codec_context_| here! With threaded decoding,
77  // it will contain unsynchronized width/height/pix_fmt values,
78  // whereas |codec_context| contains the current threads's
79  // updated width/height/pix_fmt, which can change for adaptive
80  // content.
81  VideoFrame::Format format = PixelFormatToVideoFormat(codec_context->pix_fmt);
82  if (format == VideoFrame::UNKNOWN)
83    return AVERROR(EINVAL);
84  DCHECK(format == VideoFrame::YV12 || format == VideoFrame::YV16 ||
85         format == VideoFrame::YV12J);
86
87  gfx::Size size(codec_context->width, codec_context->height);
88  const int ret = av_image_check_size(size.width(), size.height(), 0, NULL);
89  if (ret < 0)
90    return ret;
91
92  gfx::Size natural_size;
93  if (codec_context->sample_aspect_ratio.num > 0) {
94    natural_size = GetNaturalSize(size,
95                                  codec_context->sample_aspect_ratio.num,
96                                  codec_context->sample_aspect_ratio.den);
97  } else {
98    natural_size = config_.natural_size();
99  }
100
101  // FFmpeg has specific requirements on the allocation size of the frame.  The
102  // following logic replicates FFmpeg's allocation strategy to ensure buffers
103  // are not overread / overwritten.  See ff_init_buffer_info() for details.
104  //
105  // When lowres is non-zero, dimensions should be divided by 2^(lowres), but
106  // since we don't use this, just DCHECK that it's zero.
107  DCHECK_EQ(codec_context->lowres, 0);
108  gfx::Size coded_size(std::max(size.width(), codec_context->coded_width),
109                       std::max(size.height(), codec_context->coded_height));
110
111  if (!VideoFrame::IsValidConfig(
112          format, coded_size, gfx::Rect(size), natural_size))
113    return AVERROR(EINVAL);
114
115  scoped_refptr<VideoFrame> video_frame = frame_pool_.CreateFrame(
116      format, coded_size, gfx::Rect(size), natural_size, kNoTimestamp());
117
118  for (int i = 0; i < 3; i++) {
119    frame->data[i] = video_frame->data(i);
120    frame->linesize[i] = video_frame->stride(i);
121  }
122
123  frame->width = coded_size.width();
124  frame->height = coded_size.height();
125  frame->format = codec_context->pix_fmt;
126  frame->reordered_opaque = codec_context->reordered_opaque;
127
128  // Now create an AVBufferRef for the data just allocated. It will own the
129  // reference to the VideoFrame object.
130  void* opaque = NULL;
131  video_frame.swap(reinterpret_cast<VideoFrame**>(&opaque));
132  frame->buf[0] =
133      av_buffer_create(frame->data[0],
134                       VideoFrame::AllocationSize(format, coded_size),
135                       ReleaseVideoBufferImpl,
136                       opaque,
137                       0);
138  return 0;
139}
140
141void FFmpegVideoDecoder::Initialize(const VideoDecoderConfig& config,
142                                    bool low_delay,
143                                    const PipelineStatusCB& status_cb) {
144  DCHECK(task_runner_->BelongsToCurrentThread());
145  DCHECK(decode_cb_.is_null());
146  DCHECK(!config.is_encrypted());
147
148  FFmpegGlue::InitializeFFmpeg();
149
150  config_ = config;
151  PipelineStatusCB initialize_cb = BindToCurrentLoop(status_cb);
152
153  if (!config.IsValidConfig() || !ConfigureDecoder(low_delay)) {
154    initialize_cb.Run(DECODER_ERROR_NOT_SUPPORTED);
155    return;
156  }
157
158  // Success!
159  state_ = kNormal;
160  initialize_cb.Run(PIPELINE_OK);
161}
162
163void FFmpegVideoDecoder::Decode(const scoped_refptr<DecoderBuffer>& buffer,
164                                const DecodeCB& decode_cb) {
165  DCHECK(task_runner_->BelongsToCurrentThread());
166  DCHECK(!decode_cb.is_null());
167  CHECK_NE(state_, kUninitialized);
168  CHECK(decode_cb_.is_null()) << "Overlapping decodes are not supported.";
169  decode_cb_ = BindToCurrentLoop(decode_cb);
170
171  if (state_ == kError) {
172    base::ResetAndReturn(&decode_cb_).Run(kDecodeError, NULL);
173    return;
174  }
175
176  // Return empty frames if decoding has finished.
177  if (state_ == kDecodeFinished) {
178    base::ResetAndReturn(&decode_cb_).Run(kOk, VideoFrame::CreateEOSFrame());
179    return;
180  }
181
182  DecodeBuffer(buffer);
183}
184
185void FFmpegVideoDecoder::Reset(const base::Closure& closure) {
186  DCHECK(task_runner_->BelongsToCurrentThread());
187  DCHECK(decode_cb_.is_null());
188
189  avcodec_flush_buffers(codec_context_.get());
190  state_ = kNormal;
191  task_runner_->PostTask(FROM_HERE, closure);
192}
193
194void FFmpegVideoDecoder::Stop() {
195  DCHECK(task_runner_->BelongsToCurrentThread());
196
197  if (state_ == kUninitialized)
198    return;
199
200  ReleaseFFmpegResources();
201  state_ = kUninitialized;
202}
203
204FFmpegVideoDecoder::~FFmpegVideoDecoder() {
205  DCHECK_EQ(kUninitialized, state_);
206  DCHECK(!codec_context_);
207  DCHECK(!av_frame_);
208}
209
210void FFmpegVideoDecoder::DecodeBuffer(
211    const scoped_refptr<DecoderBuffer>& buffer) {
212  DCHECK(task_runner_->BelongsToCurrentThread());
213  DCHECK_NE(state_, kUninitialized);
214  DCHECK_NE(state_, kDecodeFinished);
215  DCHECK_NE(state_, kError);
216  DCHECK(!decode_cb_.is_null());
217  DCHECK(buffer);
218
219  // During decode, because reads are issued asynchronously, it is possible to
220  // receive multiple end of stream buffers since each decode is acked. When the
221  // first end of stream buffer is read, FFmpeg may still have frames queued
222  // up in the decoder so we need to go through the decode loop until it stops
223  // giving sensible data.  After that, the decoder should output empty
224  // frames.  There are three states the decoder can be in:
225  //
226  //   kNormal: This is the starting state. Buffers are decoded. Decode errors
227  //            are discarded.
228  //   kFlushCodec: There isn't any more input data. Call avcodec_decode_video2
229  //                until no more data is returned to flush out remaining
230  //                frames. The input buffer is ignored at this point.
231  //   kDecodeFinished: All calls return empty frames.
232  //   kError: Unexpected error happened.
233  //
234  // These are the possible state transitions.
235  //
236  // kNormal -> kFlushCodec:
237  //     When buffer->end_of_stream() is first true.
238  // kNormal -> kError:
239  //     A decoding error occurs and decoding needs to stop.
240  // kFlushCodec -> kDecodeFinished:
241  //     When avcodec_decode_video2() returns 0 data.
242  // kFlushCodec -> kError:
243  //     When avcodec_decode_video2() errors out.
244  // (any state) -> kNormal:
245  //     Any time Reset() is called.
246
247  // Transition to kFlushCodec on the first end of stream buffer.
248  if (state_ == kNormal && buffer->end_of_stream()) {
249    state_ = kFlushCodec;
250  }
251
252  scoped_refptr<VideoFrame> video_frame;
253  if (!FFmpegDecode(buffer, &video_frame)) {
254    state_ = kError;
255    base::ResetAndReturn(&decode_cb_).Run(kDecodeError, NULL);
256    return;
257  }
258
259  if (!video_frame.get()) {
260    if (state_ == kFlushCodec) {
261      DCHECK(buffer->end_of_stream());
262      state_ = kDecodeFinished;
263      base::ResetAndReturn(&decode_cb_)
264          .Run(kOk, VideoFrame::CreateEOSFrame());
265      return;
266    }
267
268    base::ResetAndReturn(&decode_cb_).Run(kNotEnoughData, NULL);
269    return;
270  }
271
272  base::ResetAndReturn(&decode_cb_).Run(kOk, video_frame);
273}
274
275bool FFmpegVideoDecoder::FFmpegDecode(
276    const scoped_refptr<DecoderBuffer>& buffer,
277    scoped_refptr<VideoFrame>* video_frame) {
278  DCHECK(video_frame);
279
280  // Create a packet for input data.
281  // Due to FFmpeg API changes we no longer have const read-only pointers.
282  AVPacket packet;
283  av_init_packet(&packet);
284  if (buffer->end_of_stream()) {
285    packet.data = NULL;
286    packet.size = 0;
287  } else {
288    packet.data = const_cast<uint8*>(buffer->data());
289    packet.size = buffer->data_size();
290
291    // Let FFmpeg handle presentation timestamp reordering.
292    codec_context_->reordered_opaque = buffer->timestamp().InMicroseconds();
293  }
294
295  int frame_decoded = 0;
296  int result = avcodec_decode_video2(codec_context_.get(),
297                                     av_frame_.get(),
298                                     &frame_decoded,
299                                     &packet);
300  // Log the problem if we can't decode a video frame and exit early.
301  if (result < 0) {
302    LOG(ERROR) << "Error decoding video: " << buffer->AsHumanReadableString();
303    *video_frame = NULL;
304    return false;
305  }
306
307  // FFmpeg says some codecs might have multiple frames per packet.  Previous
308  // discussions with rbultje@ indicate this shouldn't be true for the codecs
309  // we use.
310  DCHECK_EQ(result, packet.size);
311
312  // If no frame was produced then signal that more data is required to
313  // produce more frames. This can happen under two circumstances:
314  //   1) Decoder was recently initialized/flushed
315  //   2) End of stream was reached and all internal frames have been output
316  if (frame_decoded == 0) {
317    *video_frame = NULL;
318    return true;
319  }
320
321  // TODO(fbarchard): Work around for FFmpeg http://crbug.com/27675
322  // The decoder is in a bad state and not decoding correctly.
323  // Checking for NULL avoids a crash in CopyPlane().
324  if (!av_frame_->data[VideoFrame::kYPlane] ||
325      !av_frame_->data[VideoFrame::kUPlane] ||
326      !av_frame_->data[VideoFrame::kVPlane]) {
327    LOG(ERROR) << "Video frame was produced yet has invalid frame data.";
328    *video_frame = NULL;
329    av_frame_unref(av_frame_.get());
330    return false;
331  }
332
333  *video_frame =
334      reinterpret_cast<VideoFrame*>(av_buffer_get_opaque(av_frame_->buf[0]));
335
336  (*video_frame)->set_timestamp(
337      base::TimeDelta::FromMicroseconds(av_frame_->reordered_opaque));
338
339  av_frame_unref(av_frame_.get());
340  return true;
341}
342
343void FFmpegVideoDecoder::ReleaseFFmpegResources() {
344  codec_context_.reset();
345  av_frame_.reset();
346}
347
348bool FFmpegVideoDecoder::ConfigureDecoder(bool low_delay) {
349  // Release existing decoder resources if necessary.
350  ReleaseFFmpegResources();
351
352  // Initialize AVCodecContext structure.
353  codec_context_.reset(avcodec_alloc_context3(NULL));
354  VideoDecoderConfigToAVCodecContext(config_, codec_context_.get());
355
356  codec_context_->thread_count = GetThreadCount(codec_context_->codec_id);
357  codec_context_->thread_type = low_delay ? FF_THREAD_SLICE : FF_THREAD_FRAME;
358  codec_context_->opaque = this;
359  codec_context_->flags |= CODEC_FLAG_EMU_EDGE;
360  codec_context_->get_buffer2 = GetVideoBufferImpl;
361  codec_context_->refcounted_frames = 1;
362
363  AVCodec* codec = avcodec_find_decoder(codec_context_->codec_id);
364  if (!codec || avcodec_open2(codec_context_.get(), codec, NULL) < 0) {
365    ReleaseFFmpegResources();
366    return false;
367  }
368
369  av_frame_.reset(av_frame_alloc());
370  return true;
371}
372
373}  // namespace media
374