1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/common/gpu/media/android_video_encode_accelerator.h"
6
7#include "base/bind.h"
8#include "base/command_line.h"
9#include "base/logging.h"
10#include "base/message_loop/message_loop.h"
11#include "base/metrics/histogram.h"
12#include "content/common/gpu/gpu_channel.h"
13#include "content/public/common/content_switches.h"
14#include "gpu/command_buffer/service/gles2_cmd_decoder.h"
15#include "media/base/android/media_codec_bridge.h"
16#include "media/base/bitstream_buffer.h"
17#include "media/base/limits.h"
18#include "media/video/picture.h"
19#include "third_party/libyuv/include/libyuv/convert_from.h"
20#include "ui/gl/android/scoped_java_surface.h"
21#include "ui/gl/gl_bindings.h"
22
23using media::MediaCodecBridge;
24using media::VideoCodecBridge;
25using media::VideoFrame;
26
27namespace content {
28
29enum {
30  // Subset of MediaCodecInfo.CodecCapabilities.
31  COLOR_FORMAT_YUV420_SEMIPLANAR = 21,
32};
33
34// Helper macros for dealing with failure.  If |result| evaluates false, emit
35// |log| to DLOG(ERROR), register |error| with the client, and return.
36#define RETURN_ON_FAILURE(result, log, error)                  \
37  do {                                                         \
38    if (!(result)) {                                           \
39      DLOG(ERROR) << log;                                      \
40      if (client_ptr_factory_->GetWeakPtr()) {                 \
41        client_ptr_factory_->GetWeakPtr()->NotifyError(error); \
42        client_ptr_factory_.reset();                           \
43      }                                                        \
44      return;                                                  \
45    }                                                          \
46  } while (0)
47
48// Because MediaCodec is thread-hostile (must be poked on a single thread) and
49// has no callback mechanism (b/11990118), we must drive it by polling for
50// complete frames (and available input buffers, when the codec is fully
51// saturated).  This function defines the polling delay.  The value used is an
52// arbitrary choice that trades off CPU utilization (spinning) against latency.
53// Mirrors android_video_decode_accelerator.cc::DecodePollDelay().
54static inline const base::TimeDelta EncodePollDelay() {
55  // An alternative to this polling scheme could be to dedicate a new thread
56  // (instead of using the ChildThread) to run the MediaCodec, and make that
57  // thread use the timeout-based flavor of MediaCodec's dequeue methods when it
58  // believes the codec should complete "soon" (e.g. waiting for an input
59  // buffer, or waiting for a picture when it knows enough complete input
60  // pictures have been fed to saturate any internal buffering).  This is
61  // speculative and it's unclear that this would be a win (nor that there's a
62  // reasonably device-agnostic way to fill in the "believes" above).
63  return base::TimeDelta::FromMilliseconds(10);
64}
65
66static inline const base::TimeDelta NoWaitTimeOut() {
67  return base::TimeDelta::FromMicroseconds(0);
68}
69
70AndroidVideoEncodeAccelerator::AndroidVideoEncodeAccelerator()
71    : num_buffers_at_codec_(0),
72      num_output_buffers_(-1),
73      output_buffers_capacity_(0),
74      last_set_bitrate_(0) {}
75
76AndroidVideoEncodeAccelerator::~AndroidVideoEncodeAccelerator() {
77  DCHECK(thread_checker_.CalledOnValidThread());
78}
79
80// static
81std::vector<media::VideoEncodeAccelerator::SupportedProfile>
82AndroidVideoEncodeAccelerator::GetSupportedProfiles() {
83  std::vector<MediaCodecBridge::CodecsInfo> codecs_info =
84      MediaCodecBridge::GetCodecsInfo();
85
86  std::vector<SupportedProfile> profiles;
87
88#if defined(ENABLE_WEBRTC)
89  const CommandLine* cmd_line = CommandLine::ForCurrentProcess();
90  if (cmd_line->HasSwitch(switches::kDisableWebRtcHWEncoding))
91    return profiles;
92#endif
93
94  for (size_t i = 0; i < codecs_info.size(); ++i) {
95    const MediaCodecBridge::CodecsInfo& info = codecs_info[i];
96    if (info.direction != media::MEDIA_CODEC_ENCODER || info.codecs != "vp8" ||
97        VideoCodecBridge::IsKnownUnaccelerated(media::kCodecVP8,
98                                               media::MEDIA_CODEC_ENCODER)) {
99      // We're only looking for a HW VP8 encoder.
100      continue;
101    }
102    SupportedProfile profile;
103    profile.profile = media::VP8PROFILE_MAIN;
104    // Wouldn't it be nice if MediaCodec exposed the maximum capabilities of the
105    // encoder?  Sure would be.  Too bad it doesn't.  So we hard-code some
106    // reasonable defaults.
107    profile.max_resolution.SetSize(1920, 1088);
108    profile.max_framerate.numerator = 30;
109    profile.max_framerate.denominator = 1;
110    profiles.push_back(profile);
111  }
112  return profiles;
113}
114
115bool AndroidVideoEncodeAccelerator::Initialize(
116    VideoFrame::Format format,
117    const gfx::Size& input_visible_size,
118    media::VideoCodecProfile output_profile,
119    uint32 initial_bitrate,
120    Client* client) {
121  DVLOG(3) << __PRETTY_FUNCTION__ << " format: " << format
122           << ", input_visible_size: " << input_visible_size.ToString()
123           << ", output_profile: " << output_profile
124           << ", initial_bitrate: " << initial_bitrate;
125  DCHECK(!media_codec_);
126  DCHECK(thread_checker_.CalledOnValidThread());
127
128  client_ptr_factory_.reset(new base::WeakPtrFactory<Client>(client));
129
130  if (!(media::MediaCodecBridge::SupportsSetParameters() &&
131        format == VideoFrame::I420 &&
132        output_profile == media::VP8PROFILE_MAIN)) {
133    DLOG(ERROR) << "Unexpected combo: " << format << ", " << output_profile;
134    return false;
135  }
136
137  last_set_bitrate_ = initial_bitrate;
138
139  // Only consider using MediaCodec if it's likely backed by hardware.
140  if (media::VideoCodecBridge::IsKnownUnaccelerated(
141          media::kCodecVP8, media::MEDIA_CODEC_ENCODER)) {
142    DLOG(ERROR) << "No HW support";
143    return false;
144  }
145
146  // TODO(fischman): when there is more HW out there with different color-space
147  // support, this should turn into a negotiation with the codec for supported
148  // formats.  For now we use the only format supported by the only available
149  // HW.
150  media_codec_.reset(
151      media::VideoCodecBridge::CreateEncoder(media::kCodecVP8,
152                                             input_visible_size,
153                                             initial_bitrate,
154                                             INITIAL_FRAMERATE,
155                                             IFRAME_INTERVAL,
156                                             COLOR_FORMAT_YUV420_SEMIPLANAR));
157
158  if (!media_codec_) {
159    DLOG(ERROR) << "Failed to create/start the codec: "
160                << input_visible_size.ToString();
161    return false;
162  }
163
164  num_output_buffers_ = media_codec_->GetOutputBuffersCount();
165  output_buffers_capacity_ = media_codec_->GetOutputBuffersCapacity();
166  base::MessageLoop::current()->PostTask(
167      FROM_HERE,
168      base::Bind(&VideoEncodeAccelerator::Client::RequireBitstreamBuffers,
169                 client_ptr_factory_->GetWeakPtr(),
170                 num_output_buffers_,
171                 input_visible_size,
172                 output_buffers_capacity_));
173  return true;
174}
175
176void AndroidVideoEncodeAccelerator::MaybeStartIOTimer() {
177  if (!io_timer_.IsRunning() &&
178      (num_buffers_at_codec_ > 0 || !pending_frames_.empty())) {
179    io_timer_.Start(FROM_HERE,
180                    EncodePollDelay(),
181                    this,
182                    &AndroidVideoEncodeAccelerator::DoIOTask);
183  }
184}
185
186void AndroidVideoEncodeAccelerator::MaybeStopIOTimer() {
187  if (io_timer_.IsRunning() &&
188      (num_buffers_at_codec_ == 0 && pending_frames_.empty())) {
189    io_timer_.Stop();
190  }
191}
192
193void AndroidVideoEncodeAccelerator::Encode(
194    const scoped_refptr<VideoFrame>& frame,
195    bool force_keyframe) {
196  DVLOG(3) << __PRETTY_FUNCTION__ << ": " << force_keyframe;
197  DCHECK(thread_checker_.CalledOnValidThread());
198  RETURN_ON_FAILURE(frame->format() == VideoFrame::I420,
199                    "Unexpected format",
200                    kInvalidArgumentError);
201
202  // MediaCodec doesn't have a way to specify stride for non-Packed formats, so
203  // we insist on being called with packed frames and no cropping :(
204  RETURN_ON_FAILURE(frame->row_bytes(VideoFrame::kYPlane) ==
205                            frame->stride(VideoFrame::kYPlane) &&
206                        frame->row_bytes(VideoFrame::kUPlane) ==
207                            frame->stride(VideoFrame::kUPlane) &&
208                        frame->row_bytes(VideoFrame::kVPlane) ==
209                            frame->stride(VideoFrame::kVPlane) &&
210                        frame->coded_size() == frame->visible_rect().size(),
211                    "Non-packed frame, or visible_rect != coded_size",
212                    kInvalidArgumentError);
213
214  pending_frames_.push(MakeTuple(frame, force_keyframe, base::Time::Now()));
215  DoIOTask();
216}
217
218void AndroidVideoEncodeAccelerator::UseOutputBitstreamBuffer(
219    const media::BitstreamBuffer& buffer) {
220  DVLOG(3) << __PRETTY_FUNCTION__ << ": bitstream_buffer_id=" << buffer.id();
221  DCHECK(thread_checker_.CalledOnValidThread());
222  RETURN_ON_FAILURE(buffer.size() >= media_codec_->GetOutputBuffersCapacity(),
223                    "Output buffers too small!",
224                    kInvalidArgumentError);
225  available_bitstream_buffers_.push_back(buffer);
226  DoIOTask();
227}
228
229void AndroidVideoEncodeAccelerator::RequestEncodingParametersChange(
230    uint32 bitrate,
231    uint32 framerate) {
232  DVLOG(3) << __PRETTY_FUNCTION__ << ": bitrate: " << bitrate
233           << ", framerate: " << framerate;
234  DCHECK(thread_checker_.CalledOnValidThread());
235  if (bitrate != last_set_bitrate_) {
236    last_set_bitrate_ = bitrate;
237    media_codec_->SetVideoBitrate(bitrate);
238  }
239  // Note: Android's MediaCodec doesn't allow mid-stream adjustments to
240  // framerate, so we ignore that here.  This is OK because Android only uses
241  // the framerate value from MediaFormat during configure() as a proxy for
242  // bitrate, and we set that explicitly.
243}
244
245void AndroidVideoEncodeAccelerator::Destroy() {
246  DVLOG(3) << __PRETTY_FUNCTION__;
247  DCHECK(thread_checker_.CalledOnValidThread());
248  client_ptr_factory_.reset();
249  if (media_codec_) {
250    if (io_timer_.IsRunning())
251      io_timer_.Stop();
252    media_codec_->Stop();
253  }
254  delete this;
255}
256
257void AndroidVideoEncodeAccelerator::DoIOTask() {
258  QueueInput();
259  DequeueOutput();
260  MaybeStartIOTimer();
261  MaybeStopIOTimer();
262}
263
264void AndroidVideoEncodeAccelerator::QueueInput() {
265  if (!client_ptr_factory_->GetWeakPtr() || pending_frames_.empty())
266    return;
267
268  int input_buf_index = 0;
269  media::MediaCodecStatus status =
270      media_codec_->DequeueInputBuffer(NoWaitTimeOut(), &input_buf_index);
271  if (status != media::MEDIA_CODEC_OK) {
272    DCHECK(status == media::MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER ||
273           status == media::MEDIA_CODEC_ERROR);
274    RETURN_ON_FAILURE(status != media::MEDIA_CODEC_ERROR,
275                      "MediaCodec error",
276                      kPlatformFailureError);
277    return;
278  }
279
280  const PendingFrames::value_type& input = pending_frames_.front();
281  bool is_key_frame = input.b;
282  if (is_key_frame) {
283    // Ideally MediaCodec would honor BUFFER_FLAG_SYNC_FRAME so we could
284    // indicate this in the QueueInputBuffer() call below and guarantee _this_
285    // frame be encoded as a key frame, but sadly that flag is ignored.
286    // Instead, we request a key frame "soon".
287    media_codec_->RequestKeyFrameSoon();
288  }
289  scoped_refptr<VideoFrame> frame = input.a;
290
291  uint8* buffer = NULL;
292  size_t capacity = 0;
293  media_codec_->GetInputBuffer(input_buf_index, &buffer, &capacity);
294
295  size_t queued_size =
296      VideoFrame::AllocationSize(VideoFrame::I420, frame->coded_size());
297  RETURN_ON_FAILURE(capacity >= queued_size,
298                    "Failed to get input buffer: " << input_buf_index,
299                    kPlatformFailureError);
300
301  uint8* dst_y = buffer;
302  int dst_stride_y = frame->stride(VideoFrame::kYPlane);
303  uint8* dst_uv = buffer + frame->stride(VideoFrame::kYPlane) *
304                               frame->rows(VideoFrame::kYPlane);
305  int dst_stride_uv = frame->stride(VideoFrame::kUPlane) * 2;
306  // Why NV12?  Because COLOR_FORMAT_YUV420_SEMIPLANAR.  See comment at other
307  // mention of that constant.
308  bool converted = !libyuv::I420ToNV12(frame->data(VideoFrame::kYPlane),
309                                       frame->stride(VideoFrame::kYPlane),
310                                       frame->data(VideoFrame::kUPlane),
311                                       frame->stride(VideoFrame::kUPlane),
312                                       frame->data(VideoFrame::kVPlane),
313                                       frame->stride(VideoFrame::kVPlane),
314                                       dst_y,
315                                       dst_stride_y,
316                                       dst_uv,
317                                       dst_stride_uv,
318                                       frame->coded_size().width(),
319                                       frame->coded_size().height());
320  RETURN_ON_FAILURE(converted, "Failed to I420ToNV12!", kPlatformFailureError);
321
322  fake_input_timestamp_ += base::TimeDelta::FromMicroseconds(1);
323  status = media_codec_->QueueInputBuffer(
324      input_buf_index, NULL, queued_size, fake_input_timestamp_);
325  UMA_HISTOGRAM_TIMES("Media.AVEA.InputQueueTime", base::Time::Now() - input.c);
326  RETURN_ON_FAILURE(status == media::MEDIA_CODEC_OK,
327                    "Failed to QueueInputBuffer: " << status,
328                    kPlatformFailureError);
329  ++num_buffers_at_codec_;
330  pending_frames_.pop();
331}
332
333bool AndroidVideoEncodeAccelerator::DoOutputBuffersSuffice() {
334  // If this returns false ever, then the VEA::Client interface will need to
335  // grow a DismissBitstreamBuffer() call, and VEA::Client impls will have to be
336  // prepared to field multiple requests to RequireBitstreamBuffers().
337  int count = media_codec_->GetOutputBuffersCount();
338  size_t capacity = media_codec_->GetOutputBuffersCapacity();
339  bool ret = media_codec_->GetOutputBuffers() && count <= num_output_buffers_ &&
340             capacity <= output_buffers_capacity_;
341  LOG_IF(ERROR, !ret) << "Need more/bigger buffers; before: "
342                      << num_output_buffers_ << "x" << output_buffers_capacity_
343                      << ", now: " << count << "x" << capacity;
344  UMA_HISTOGRAM_BOOLEAN("Media.AVEA.OutputBuffersSuffice", ret);
345  return ret;
346}
347
348void AndroidVideoEncodeAccelerator::DequeueOutput() {
349  if (!client_ptr_factory_->GetWeakPtr() ||
350      available_bitstream_buffers_.empty() || num_buffers_at_codec_ == 0) {
351    return;
352  }
353
354  int32 buf_index = 0;
355  size_t offset = 0;
356  size_t size = 0;
357  bool key_frame = false;
358  do {
359    media::MediaCodecStatus status = media_codec_->DequeueOutputBuffer(
360        NoWaitTimeOut(), &buf_index, &offset, &size, NULL, NULL, &key_frame);
361    switch (status) {
362      case media::MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
363        return;
364
365      case media::MEDIA_CODEC_ERROR:
366        RETURN_ON_FAILURE(false, "Codec error", kPlatformFailureError);
367        // Unreachable because of previous statement, but included for clarity.
368        return;
369
370      case media::MEDIA_CODEC_OUTPUT_FORMAT_CHANGED:  // Fall-through.
371      case media::MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED:
372        RETURN_ON_FAILURE(DoOutputBuffersSuffice(),
373                          "Bitstream now requires more/larger buffers",
374                          kPlatformFailureError);
375        break;
376
377      case media::MEDIA_CODEC_OK:
378        DCHECK_GE(buf_index, 0);
379        break;
380
381      default:
382        NOTREACHED();
383        break;
384    }
385  } while (buf_index < 0);
386
387  media::BitstreamBuffer bitstream_buffer = available_bitstream_buffers_.back();
388  available_bitstream_buffers_.pop_back();
389  scoped_ptr<base::SharedMemory> shm(
390      new base::SharedMemory(bitstream_buffer.handle(), false));
391  RETURN_ON_FAILURE(shm->Map(bitstream_buffer.size()),
392                    "Failed to map SHM",
393                    kPlatformFailureError);
394  RETURN_ON_FAILURE(size <= shm->mapped_size(),
395                    "Encoded buffer too large: " << size << ">"
396                                                 << shm->mapped_size(),
397                    kPlatformFailureError);
398
399  media_codec_->CopyFromOutputBuffer(buf_index, offset, shm->memory(), size);
400  media_codec_->ReleaseOutputBuffer(buf_index, false);
401  --num_buffers_at_codec_;
402
403  UMA_HISTOGRAM_COUNTS_10000("Media.AVEA.EncodedBufferSizeKB", size / 1024);
404  base::MessageLoop::current()->PostTask(
405      FROM_HERE,
406      base::Bind(&VideoEncodeAccelerator::Client::BitstreamBufferReady,
407                 client_ptr_factory_->GetWeakPtr(),
408                 bitstream_buffer.id(),
409                 size,
410                 key_frame));
411}
412
413}  // namespace content
414