1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "media/filters/ffmpeg_video_decoder.h"
6
7#include <algorithm>
8#include <string>
9
10#include "base/bind.h"
11#include "base/callback_helpers.h"
12#include "base/command_line.h"
13#include "base/location.h"
14#include "base/single_thread_task_runner.h"
15#include "base/strings/string_number_conversions.h"
16#include "media/base/bind_to_current_loop.h"
17#include "media/base/decoder_buffer.h"
18#include "media/base/limits.h"
19#include "media/base/media_switches.h"
20#include "media/base/pipeline.h"
21#include "media/base/video_decoder_config.h"
22#include "media/base/video_frame.h"
23#include "media/base/video_util.h"
24#include "media/ffmpeg/ffmpeg_common.h"
25#include "media/filters/ffmpeg_glue.h"
26
27namespace media {
28
29// Always try to use three threads for video decoding.  There is little reason
30// not to since current day CPUs tend to be multi-core and we measured
31// performance benefits on older machines such as P4s with hyperthreading.
32//
33// Handling decoding on separate threads also frees up the pipeline thread to
34// continue processing. Although it'd be nice to have the option of a single
35// decoding thread, FFmpeg treats having one thread the same as having zero
36// threads (i.e., avcodec_decode_video() will execute on the calling thread).
37// Yet another reason for having two threads :)
38static const int kDecodeThreads = 2;
39static const int kMaxDecodeThreads = 16;
40
41// Returns the number of threads given the FFmpeg CodecID. Also inspects the
42// command line for a valid --video-threads flag.
43static int GetThreadCount(AVCodecID codec_id) {
44  // Refer to http://crbug.com/93932 for tsan suppressions on decoding.
45  int decode_threads = kDecodeThreads;
46
47  const CommandLine* cmd_line = CommandLine::ForCurrentProcess();
48  std::string threads(cmd_line->GetSwitchValueASCII(switches::kVideoThreads));
49  if (threads.empty() || !base::StringToInt(threads, &decode_threads))
50    return decode_threads;
51
52  decode_threads = std::max(decode_threads, 0);
53  decode_threads = std::min(decode_threads, kMaxDecodeThreads);
54  return decode_threads;
55}
56
57static int GetVideoBufferImpl(struct AVCodecContext* s,
58                              AVFrame* frame,
59                              int flags) {
60  FFmpegVideoDecoder* decoder = static_cast<FFmpegVideoDecoder*>(s->opaque);
61  return decoder->GetVideoBuffer(s, frame, flags);
62}
63
64static void ReleaseVideoBufferImpl(void* opaque, uint8* data) {
65  scoped_refptr<VideoFrame> video_frame;
66  video_frame.swap(reinterpret_cast<VideoFrame**>(&opaque));
67}
68
69static size_t RoundUp(size_t value, size_t alignment) {
70  // Check that |alignment| is a power of 2.
71  DCHECK((alignment + (alignment - 1)) == (alignment | (alignment - 1)));
72  return ((value + (alignment - 1)) & ~(alignment - 1));
73}
74
75FFmpegVideoDecoder::FFmpegVideoDecoder(
76    const scoped_refptr<base::SingleThreadTaskRunner>& task_runner)
77    : task_runner_(task_runner), state_(kUninitialized),
78      decode_nalus_(false) {}
79
80int FFmpegVideoDecoder::GetVideoBuffer(struct AVCodecContext* codec_context,
81                                       AVFrame* frame,
82                                       int flags) {
83  // Don't use |codec_context_| here! With threaded decoding,
84  // it will contain unsynchronized width/height/pix_fmt values,
85  // whereas |codec_context| contains the current threads's
86  // updated width/height/pix_fmt, which can change for adaptive
87  // content.
88  VideoFrame::Format format = PixelFormatToVideoFormat(codec_context->pix_fmt);
89  if (format == VideoFrame::UNKNOWN)
90    return AVERROR(EINVAL);
91  DCHECK(format == VideoFrame::YV12 || format == VideoFrame::YV16 ||
92         format == VideoFrame::YV12J || format == VideoFrame::YV24);
93
94  gfx::Size size(codec_context->width, codec_context->height);
95  const int ret = av_image_check_size(size.width(), size.height(), 0, NULL);
96  if (ret < 0)
97    return ret;
98
99  gfx::Size natural_size;
100  if (codec_context->sample_aspect_ratio.num > 0) {
101    natural_size = GetNaturalSize(size,
102                                  codec_context->sample_aspect_ratio.num,
103                                  codec_context->sample_aspect_ratio.den);
104  } else {
105    natural_size = config_.natural_size();
106  }
107
108  // FFmpeg has specific requirements on the allocation size of the frame.  The
109  // following logic replicates FFmpeg's allocation strategy to ensure buffers
110  // are not overread / overwritten.  See ff_init_buffer_info() for details.
111  //
112  // When lowres is non-zero, dimensions should be divided by 2^(lowres), but
113  // since we don't use this, just DCHECK that it's zero.
114  //
115  // Always round up to a multiple of two to match VideoFrame restrictions on
116  // frame alignment.
117  DCHECK_EQ(codec_context->lowres, 0);
118  gfx::Size coded_size(
119      RoundUp(std::max(size.width(), codec_context->coded_width), 2),
120      RoundUp(std::max(size.height(), codec_context->coded_height), 2));
121
122  if (!VideoFrame::IsValidConfig(
123          format, coded_size, gfx::Rect(size), natural_size))
124    return AVERROR(EINVAL);
125
126  scoped_refptr<VideoFrame> video_frame = frame_pool_.CreateFrame(
127      format, coded_size, gfx::Rect(size), natural_size, kNoTimestamp());
128
129  for (int i = 0; i < 3; i++) {
130    frame->data[i] = video_frame->data(i);
131    frame->linesize[i] = video_frame->stride(i);
132  }
133
134  frame->width = coded_size.width();
135  frame->height = coded_size.height();
136  frame->format = codec_context->pix_fmt;
137  frame->reordered_opaque = codec_context->reordered_opaque;
138
139  // Now create an AVBufferRef for the data just allocated. It will own the
140  // reference to the VideoFrame object.
141  void* opaque = NULL;
142  video_frame.swap(reinterpret_cast<VideoFrame**>(&opaque));
143  frame->buf[0] =
144      av_buffer_create(frame->data[0],
145                       VideoFrame::AllocationSize(format, coded_size),
146                       ReleaseVideoBufferImpl,
147                       opaque,
148                       0);
149  return 0;
150}
151
152std::string FFmpegVideoDecoder::GetDisplayName() const {
153  return "FFmpegVideoDecoder";
154}
155
156void FFmpegVideoDecoder::Initialize(const VideoDecoderConfig& config,
157                                    bool low_delay,
158                                    const PipelineStatusCB& status_cb,
159                                    const OutputCB& output_cb) {
160  DCHECK(task_runner_->BelongsToCurrentThread());
161  DCHECK(!config.is_encrypted());
162  DCHECK(!output_cb.is_null());
163
164  FFmpegGlue::InitializeFFmpeg();
165
166  config_ = config;
167  PipelineStatusCB initialize_cb = BindToCurrentLoop(status_cb);
168
169  if (!config.IsValidConfig() || !ConfigureDecoder(low_delay)) {
170    initialize_cb.Run(DECODER_ERROR_NOT_SUPPORTED);
171    return;
172  }
173
174  output_cb_ = BindToCurrentLoop(output_cb);
175
176  // Success!
177  state_ = kNormal;
178  initialize_cb.Run(PIPELINE_OK);
179}
180
181void FFmpegVideoDecoder::Decode(const scoped_refptr<DecoderBuffer>& buffer,
182                                const DecodeCB& decode_cb) {
183  DCHECK(task_runner_->BelongsToCurrentThread());
184  DCHECK(buffer.get());
185  DCHECK(!decode_cb.is_null());
186  CHECK_NE(state_, kUninitialized);
187
188  DecodeCB decode_cb_bound = BindToCurrentLoop(decode_cb);
189
190  if (state_ == kError) {
191    decode_cb_bound.Run(kDecodeError);
192    return;
193  }
194
195  if (state_ == kDecodeFinished) {
196    decode_cb_bound.Run(kOk);
197    return;
198  }
199
200  DCHECK_EQ(state_, kNormal);
201
202  // During decode, because reads are issued asynchronously, it is possible to
203  // receive multiple end of stream buffers since each decode is acked. When the
204  // first end of stream buffer is read, FFmpeg may still have frames queued
205  // up in the decoder so we need to go through the decode loop until it stops
206  // giving sensible data.  After that, the decoder should output empty
207  // frames.  There are three states the decoder can be in:
208  //
209  //   kNormal: This is the starting state. Buffers are decoded. Decode errors
210  //            are discarded.
211  //   kDecodeFinished: All calls return empty frames.
212  //   kError: Unexpected error happened.
213  //
214  // These are the possible state transitions.
215  //
216  // kNormal -> kDecodeFinished:
217  //     When EOS buffer is received and the codec has been flushed.
218  // kNormal -> kError:
219  //     A decoding error occurs and decoding needs to stop.
220  // (any state) -> kNormal:
221  //     Any time Reset() is called.
222
223  bool has_produced_frame;
224  do {
225    has_produced_frame = false;
226    if (!FFmpegDecode(buffer, &has_produced_frame)) {
227      state_ = kError;
228      decode_cb_bound.Run(kDecodeError);
229      return;
230    }
231    // Repeat to flush the decoder after receiving EOS buffer.
232  } while (buffer->end_of_stream() && has_produced_frame);
233
234  if (buffer->end_of_stream())
235    state_ = kDecodeFinished;
236
237  decode_cb_bound.Run(kOk);
238}
239
240void FFmpegVideoDecoder::Reset(const base::Closure& closure) {
241  DCHECK(task_runner_->BelongsToCurrentThread());
242
243  avcodec_flush_buffers(codec_context_.get());
244  state_ = kNormal;
245  task_runner_->PostTask(FROM_HERE, closure);
246}
247
248FFmpegVideoDecoder::~FFmpegVideoDecoder() {
249  DCHECK(task_runner_->BelongsToCurrentThread());
250
251  if (state_ != kUninitialized)
252    ReleaseFFmpegResources();
253}
254
255bool FFmpegVideoDecoder::FFmpegDecode(
256    const scoped_refptr<DecoderBuffer>& buffer,
257    bool* has_produced_frame) {
258  DCHECK(!*has_produced_frame);
259
260  // Create a packet for input data.
261  // Due to FFmpeg API changes we no longer have const read-only pointers.
262  AVPacket packet;
263  av_init_packet(&packet);
264  if (buffer->end_of_stream()) {
265    packet.data = NULL;
266    packet.size = 0;
267  } else {
268    packet.data = const_cast<uint8*>(buffer->data());
269    packet.size = buffer->data_size();
270
271    // Let FFmpeg handle presentation timestamp reordering.
272    codec_context_->reordered_opaque = buffer->timestamp().InMicroseconds();
273  }
274
275  int frame_decoded = 0;
276  int result = avcodec_decode_video2(codec_context_.get(),
277                                     av_frame_.get(),
278                                     &frame_decoded,
279                                     &packet);
280  // Log the problem if we can't decode a video frame and exit early.
281  if (result < 0) {
282    LOG(ERROR) << "Error decoding video: " << buffer->AsHumanReadableString();
283    return false;
284  }
285
286  // FFmpeg says some codecs might have multiple frames per packet.  Previous
287  // discussions with rbultje@ indicate this shouldn't be true for the codecs
288  // we use.
289  DCHECK_EQ(result, packet.size);
290
291  // If no frame was produced then signal that more data is required to
292  // produce more frames. This can happen under two circumstances:
293  //   1) Decoder was recently initialized/flushed
294  //   2) End of stream was reached and all internal frames have been output
295  if (frame_decoded == 0) {
296    return true;
297  }
298
299  // TODO(fbarchard): Work around for FFmpeg http://crbug.com/27675
300  // The decoder is in a bad state and not decoding correctly.
301  // Checking for NULL avoids a crash in CopyPlane().
302  if (!av_frame_->data[VideoFrame::kYPlane] ||
303      !av_frame_->data[VideoFrame::kUPlane] ||
304      !av_frame_->data[VideoFrame::kVPlane]) {
305    LOG(ERROR) << "Video frame was produced yet has invalid frame data.";
306    av_frame_unref(av_frame_.get());
307    return false;
308  }
309
310  scoped_refptr<VideoFrame> frame =
311      reinterpret_cast<VideoFrame*>(av_buffer_get_opaque(av_frame_->buf[0]));
312  frame->set_timestamp(
313      base::TimeDelta::FromMicroseconds(av_frame_->reordered_opaque));
314  *has_produced_frame = true;
315  output_cb_.Run(frame);
316
317  av_frame_unref(av_frame_.get());
318  return true;
319}
320
321void FFmpegVideoDecoder::ReleaseFFmpegResources() {
322  codec_context_.reset();
323  av_frame_.reset();
324}
325
326bool FFmpegVideoDecoder::ConfigureDecoder(bool low_delay) {
327  // Release existing decoder resources if necessary.
328  ReleaseFFmpegResources();
329
330  // Initialize AVCodecContext structure.
331  codec_context_.reset(avcodec_alloc_context3(NULL));
332  VideoDecoderConfigToAVCodecContext(config_, codec_context_.get());
333
334  codec_context_->thread_count = GetThreadCount(codec_context_->codec_id);
335  codec_context_->thread_type = low_delay ? FF_THREAD_SLICE : FF_THREAD_FRAME;
336  codec_context_->opaque = this;
337  codec_context_->flags |= CODEC_FLAG_EMU_EDGE;
338  codec_context_->get_buffer2 = GetVideoBufferImpl;
339  codec_context_->refcounted_frames = 1;
340
341  if (decode_nalus_)
342    codec_context_->flags2 |= CODEC_FLAG2_CHUNKS;
343
344  AVCodec* codec = avcodec_find_decoder(codec_context_->codec_id);
345  if (!codec || avcodec_open2(codec_context_.get(), codec, NULL) < 0) {
346    ReleaseFFmpegResources();
347    return false;
348  }
349
350  av_frame_.reset(av_frame_alloc());
351  return true;
352}
353
354}  // namespace media
355