xref: /external/chromium_org/content/common/gpu/media/android_video_encode_accelerator.cc

// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "content/common/gpu/media/android_video_encode_accelerator.h"

#include "base/bind.h"
#include "base/command_line.h"
#include "base/logging.h"
#include "base/message_loop/message_loop.h"
#include "base/metrics/histogram.h"
#include "content/common/gpu/gpu_channel.h"
#include "content/public/common/content_switches.h"
#include "gpu/command_buffer/service/gles2_cmd_decoder.h"
#include "media/base/android/media_codec_bridge.h"
#include "media/base/bitstream_buffer.h"
#include "media/base/limits.h"
#include "media/video/picture.h"
#include "third_party/libyuv/include/libyuv/convert_from.h"
#include "ui/gl/android/scoped_java_surface.h"
#include "ui/gl/gl_bindings.h"

using media::MediaCodecBridge;
using media::VideoCodecBridge;
using media::VideoFrame;

namespace content {

enum {
  // Subset of MediaCodecInfo.CodecCapabilities.
  COLOR_FORMAT_YUV420_SEMIPLANAR = 21,
};

// Helper macros for dealing with failure.  If |result| evaluates false, emit
// |log| to DLOG(ERROR), register |error| with the client, and return.
#define RETURN_ON_FAILURE(result, log, error)                 \
  do {                                                        \
    if (!(result)) {                                          \
      DLOG(ERROR) << log;                                     \
      if (client_ptr_factory_.GetWeakPtr()) {                 \
        client_ptr_factory_.GetWeakPtr()->NotifyError(error); \
        client_ptr_factory_.InvalidateWeakPtrs();             \
      }                                                       \
      return;                                                 \
    }                                                         \
  } while (0)

// Because MediaCodec is thread-hostile (must be poked on a single thread) and
// has no callback mechanism (b/11990118), we must drive it by polling for
// complete frames (and available input buffers, when the codec is fully
// saturated).  This function defines the polling delay.  The value used is an
// arbitrary choice that trades off CPU utilization (spinning) against latency.
// Mirrors android_video_decode_accelerator.cc::DecodePollDelay().
static inline const base::TimeDelta EncodePollDelay() {
  // An alternative to this polling scheme could be to dedicate a new thread
  // (instead of using the ChildThread) to run the MediaCodec, and make that
  // thread use the timeout-based flavor of MediaCodec's dequeue methods when it
  // believes the codec should complete "soon" (e.g. waiting for an input
  // buffer, or waiting for a picture when it knows enough complete input
  // pictures have been fed to saturate any internal buffering).  This is
  // speculative and it's unclear that this would be a win (nor that there's a
  // reasonably device-agnostic way to fill in the "believes" above).
  return base::TimeDelta::FromMilliseconds(10);
}

static inline const base::TimeDelta NoWaitTimeOut() {
  return base::TimeDelta::FromMicroseconds(0);
}

AndroidVideoEncodeAccelerator::AndroidVideoEncodeAccelerator(
    media::VideoEncodeAccelerator::Client* client)
    : client_ptr_factory_(client),
      num_buffers_at_codec_(0),
      num_output_buffers_(-1),
      output_buffers_capacity_(0),
      last_set_bitrate_(0) {}

AndroidVideoEncodeAccelerator::~AndroidVideoEncodeAccelerator() {
  DCHECK(thread_checker_.CalledOnValidThread());
}

// static
std::vector<media::VideoEncodeAccelerator::SupportedProfile>
AndroidVideoEncodeAccelerator::GetSupportedProfiles() {
  std::vector<MediaCodecBridge::CodecsInfo> codecs_info =
      MediaCodecBridge::GetCodecsInfo();

  std::vector<SupportedProfile> profiles;

  const CommandLine* cmd_line = CommandLine::ForCurrentProcess();
  if (cmd_line->HasSwitch(switches::kDisableWebRtcHWEncoding))
    return profiles;

  for (size_t i = 0; i < codecs_info.size(); ++i) {
    const MediaCodecBridge::CodecsInfo& info = codecs_info[i];
    if (info.direction != media::MEDIA_CODEC_ENCODER || info.codecs != "vp8" ||
        VideoCodecBridge::IsKnownUnaccelerated(media::kCodecVP8,
                                               media::MEDIA_CODEC_ENCODER)) {
      // We're only looking for a HW VP8 encoder.
      continue;
    }
    SupportedProfile profile;
    profile.profile = media::VP8PROFILE_MAIN;
    // Wouldn't it be nice if MediaCodec exposed the maximum capabilities of the
    // encoder?  Sure would be.  Too bad it doesn't.  So we hard-code some
    // reasonable defaults.
    profile.max_resolution.SetSize(1920, 1088);
    profile.max_framerate.numerator = 30;
    profile.max_framerate.denominator = 1;
    profiles.push_back(profile);
  }
  return profiles;
}

void AndroidVideoEncodeAccelerator::Initialize(
    VideoFrame::Format format,
    const gfx::Size& input_visible_size,
    media::VideoCodecProfile output_profile,
    uint32 initial_bitrate) {
  DVLOG(3) << __PRETTY_FUNCTION__ << " format: " << format
           << ", input_visible_size: " << input_visible_size.ToString()
           << ", output_profile: " << output_profile
           << ", initial_bitrate: " << initial_bitrate;
  DCHECK(!media_codec_);
  DCHECK(thread_checker_.CalledOnValidThread());

  RETURN_ON_FAILURE(media::MediaCodecBridge::IsAvailable() &&
                        media::MediaCodecBridge::SupportsSetParameters() &&
                        format == VideoFrame::I420 &&
                        output_profile == media::VP8PROFILE_MAIN,
                    "Unexpected combo: " << format << ", " << output_profile,
                    kInvalidArgumentError);

  last_set_bitrate_ = initial_bitrate;

  // Only consider using MediaCodec if it's likely backed by hardware.
  RETURN_ON_FAILURE(!media::VideoCodecBridge::IsKnownUnaccelerated(
                         media::kCodecVP8, media::MEDIA_CODEC_ENCODER),
                    "No HW support",
                    kPlatformFailureError);

  // TODO(fischman): when there is more HW out there with different color-space
  // support, this should turn into a negotiation with the codec for supported
  // formats.  For now we use the only format supported by the only available
  // HW.
  media_codec_.reset(
      media::VideoCodecBridge::CreateEncoder(media::kCodecVP8,
                                             input_visible_size,
                                             initial_bitrate,
                                             INITIAL_FRAMERATE,
                                             IFRAME_INTERVAL,
                                             COLOR_FORMAT_YUV420_SEMIPLANAR));

  RETURN_ON_FAILURE(
      media_codec_,
      "Failed to create/start the codec: " << input_visible_size.ToString(),
      kPlatformFailureError);

  base::MessageLoop::current()->PostTask(
      FROM_HERE,
      base::Bind(&VideoEncodeAccelerator::Client::NotifyInitializeDone,
                 client_ptr_factory_.GetWeakPtr()));

  num_output_buffers_ = media_codec_->GetOutputBuffersCount();
  output_buffers_capacity_ = media_codec_->GetOutputBuffersCapacity();
  base::MessageLoop::current()->PostTask(
      FROM_HERE,
      base::Bind(&VideoEncodeAccelerator::Client::RequireBitstreamBuffers,
                 client_ptr_factory_.GetWeakPtr(),
                 num_output_buffers_,
                 input_visible_size,
                 output_buffers_capacity_));
}

void AndroidVideoEncodeAccelerator::MaybeStartIOTimer() {
  if (!io_timer_.IsRunning() &&
      (num_buffers_at_codec_ > 0 || !pending_frames_.empty())) {
    io_timer_.Start(FROM_HERE,
                    EncodePollDelay(),
                    this,
                    &AndroidVideoEncodeAccelerator::DoIOTask);
  }
}

void AndroidVideoEncodeAccelerator::MaybeStopIOTimer() {
  if (io_timer_.IsRunning() &&
      (num_buffers_at_codec_ == 0 && pending_frames_.empty())) {
    io_timer_.Stop();
  }
}

void AndroidVideoEncodeAccelerator::Encode(
    const scoped_refptr<VideoFrame>& frame,
    bool force_keyframe) {
  DVLOG(3) << __PRETTY_FUNCTION__ << ": " << force_keyframe;
  DCHECK(thread_checker_.CalledOnValidThread());
  RETURN_ON_FAILURE(frame->format() == VideoFrame::I420,
                    "Unexpected format",
                    kInvalidArgumentError);

  // MediaCodec doesn't have a way to specify stride for non-Packed formats, so
  // we insist on being called with packed frames and no cropping :(
  RETURN_ON_FAILURE(frame->row_bytes(VideoFrame::kYPlane) ==
                            frame->stride(VideoFrame::kYPlane) &&
                        frame->row_bytes(VideoFrame::kUPlane) ==
                            frame->stride(VideoFrame::kUPlane) &&
                        frame->row_bytes(VideoFrame::kVPlane) ==
                            frame->stride(VideoFrame::kVPlane) &&
                        gfx::Rect(frame->coded_size()) == frame->visible_rect(),
                    "Non-packed frame, or visible rect != coded size",
                    kInvalidArgumentError);

  pending_frames_.push(MakeTuple(frame, force_keyframe, base::Time::Now()));
  DoIOTask();
}

void AndroidVideoEncodeAccelerator::UseOutputBitstreamBuffer(
    const media::BitstreamBuffer& buffer) {
  DVLOG(3) << __PRETTY_FUNCTION__ << ": bitstream_buffer_id=" << buffer.id();
  DCHECK(thread_checker_.CalledOnValidThread());
  RETURN_ON_FAILURE(buffer.size() >= media_codec_->GetOutputBuffersCapacity(),
                    "Output buffers too small!",
                    kInvalidArgumentError);
  available_bitstream_buffers_.push_back(buffer);
  DoIOTask();
}

void AndroidVideoEncodeAccelerator::RequestEncodingParametersChange(
    uint32 bitrate,
    uint32 framerate) {
  DVLOG(3) << __PRETTY_FUNCTION__ << ": bitrate: " << bitrate
           << ", framerate: " << framerate;
  DCHECK(thread_checker_.CalledOnValidThread());
  if (bitrate != last_set_bitrate_) {
    last_set_bitrate_ = bitrate;
    media_codec_->SetVideoBitrate(bitrate);
  }
  // Note: Android's MediaCodec doesn't allow mid-stream adjustments to
  // framerate, so we ignore that here.  This is OK because Android only uses
  // the framerate value from MediaFormat during configure() as a proxy for
  // bitrate, and we set that explicitly.
}

void AndroidVideoEncodeAccelerator::Destroy() {
  DVLOG(3) << __PRETTY_FUNCTION__;
  DCHECK(thread_checker_.CalledOnValidThread());
  client_ptr_factory_.InvalidateWeakPtrs();
  if (media_codec_) {
    if (io_timer_.IsRunning())
      io_timer_.Stop();
    media_codec_->Stop();
  }
  delete this;
}

void AndroidVideoEncodeAccelerator::DoIOTask() {
  QueueInput();
  DequeueOutput();
  MaybeStartIOTimer();
  MaybeStopIOTimer();
}

void AndroidVideoEncodeAccelerator::QueueInput() {
  if (!client_ptr_factory_.GetWeakPtr() || pending_frames_.empty())
    return;

  int input_buf_index = 0;
  media::MediaCodecStatus status =
      media_codec_->DequeueInputBuffer(NoWaitTimeOut(), &input_buf_index);
  if (status != media::MEDIA_CODEC_OK) {
    DCHECK(status == media::MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER ||
           status == media::MEDIA_CODEC_ERROR);
    RETURN_ON_FAILURE(status != media::MEDIA_CODEC_ERROR,
                      "MediaCodec error",
                      kPlatformFailureError);
    return;
  }

  const PendingFrames::value_type& input = pending_frames_.front();
  bool is_key_frame = input.b;
  if (is_key_frame) {
    // Ideally MediaCodec would honor BUFFER_FLAG_SYNC_FRAME so we could
    // indicate this in the QueueInputBuffer() call below and guarantee _this_
    // frame be encoded as a key frame, but sadly that flag is ignored.
    // Instead, we request a key frame "soon".
    media_codec_->RequestKeyFrameSoon();
  }
  scoped_refptr<VideoFrame> frame = input.a;

  uint8* buffer = NULL;
  size_t capacity = 0;
  media_codec_->GetInputBuffer(input_buf_index, &buffer, &capacity);

  size_t queued_size =
      VideoFrame::AllocationSize(VideoFrame::I420, frame->coded_size());
  RETURN_ON_FAILURE(capacity >= queued_size,
                    "Failed to get input buffer: " << input_buf_index,
                    kPlatformFailureError);

  uint8* dst_y = buffer;
  int dst_stride_y = frame->stride(VideoFrame::kYPlane);
  uint8* dst_uv = buffer + frame->stride(VideoFrame::kYPlane) *
                               frame->rows(VideoFrame::kYPlane);
  int dst_stride_uv = frame->stride(VideoFrame::kUPlane) * 2;
  // Why NV12?  Because COLOR_FORMAT_YUV420_SEMIPLANAR.  See comment at other
  // mention of that constant.
  bool converted = !libyuv::I420ToNV12(frame->data(VideoFrame::kYPlane),
                                       frame->stride(VideoFrame::kYPlane),
                                       frame->data(VideoFrame::kUPlane),
                                       frame->stride(VideoFrame::kUPlane),
                                       frame->data(VideoFrame::kVPlane),
                                       frame->stride(VideoFrame::kVPlane),
                                       dst_y,
                                       dst_stride_y,
                                       dst_uv,
                                       dst_stride_uv,
                                       frame->coded_size().width(),
                                       frame->coded_size().height());
  RETURN_ON_FAILURE(converted, "Failed to I420ToNV12!", kPlatformFailureError);

  fake_input_timestamp_ += base::TimeDelta::FromMicroseconds(1);
  status = media_codec_->QueueInputBuffer(
      input_buf_index, NULL, queued_size, fake_input_timestamp_);
  UMA_HISTOGRAM_TIMES("Media.AVEA.InputQueueTime", base::Time::Now() - input.c);
  RETURN_ON_FAILURE(status == media::MEDIA_CODEC_OK,
                    "Failed to QueueInputBuffer: " << status,
                    kPlatformFailureError);
  ++num_buffers_at_codec_;
  pending_frames_.pop();
}

bool AndroidVideoEncodeAccelerator::DoOutputBuffersSuffice() {
  // If this returns false ever, then the VEA::Client interface will need to
  // grow a DismissBitstreamBuffer() call, and VEA::Client impls will have to be
  // prepared to field multiple requests to RequireBitstreamBuffers().
  int count = media_codec_->GetOutputBuffersCount();
  size_t capacity = media_codec_->GetOutputBuffersCapacity();
  bool ret = media_codec_->GetOutputBuffers() && count <= num_output_buffers_ &&
             capacity <= output_buffers_capacity_;
  LOG_IF(ERROR, !ret) << "Need more/bigger buffers; before: "
                      << num_output_buffers_ << "x" << output_buffers_capacity_
                      << ", now: " << count << "x" << capacity;
  UMA_HISTOGRAM_BOOLEAN("Media.AVEA.OutputBuffersSuffice", ret);
  return ret;
}

void AndroidVideoEncodeAccelerator::DequeueOutput() {
  if (!client_ptr_factory_.GetWeakPtr() ||
      available_bitstream_buffers_.empty() || num_buffers_at_codec_ == 0) {
    return;
  }

  int32 buf_index = 0;
  size_t offset = 0;
  size_t size = 0;
  bool key_frame = false;
  do {
    media::MediaCodecStatus status = media_codec_->DequeueOutputBuffer(
        NoWaitTimeOut(), &buf_index, &offset, &size, NULL, NULL, &key_frame);
    switch (status) {
      case media::MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
        return;

      case media::MEDIA_CODEC_ERROR:
        RETURN_ON_FAILURE(false, "Codec error", kPlatformFailureError);
        // Unreachable because of previous statement, but included for clarity.
        return;

      case media::MEDIA_CODEC_OUTPUT_FORMAT_CHANGED:  // Fall-through.
      case media::MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED:
        RETURN_ON_FAILURE(DoOutputBuffersSuffice(),
                          "Bitstream now requires more/larger buffers",
                          kPlatformFailureError);
        break;

      case media::MEDIA_CODEC_OK:
        DCHECK_GE(buf_index, 0);
        break;

      default:
        NOTREACHED();
        break;
    }
  } while (buf_index < 0);

  media::BitstreamBuffer bitstream_buffer = available_bitstream_buffers_.back();
  available_bitstream_buffers_.pop_back();
  scoped_ptr<base::SharedMemory> shm(
      new base::SharedMemory(bitstream_buffer.handle(), false));
  RETURN_ON_FAILURE(shm->Map(bitstream_buffer.size()),
                    "Failed to map SHM",
                    kPlatformFailureError);
  RETURN_ON_FAILURE(size <= shm->mapped_size(),
                    "Encoded buffer too large: " << size << ">"
                                                 << shm->mapped_size(),
                    kPlatformFailureError);

  media_codec_->CopyFromOutputBuffer(buf_index, offset, shm->memory(), size);
  media_codec_->ReleaseOutputBuffer(buf_index, false);
  --num_buffers_at_codec_;

  UMA_HISTOGRAM_COUNTS_10000("Media.AVEA.EncodedBufferSizeKB", size / 1024);
  base::MessageLoop::current()->PostTask(
      FROM_HERE,
      base::Bind(&VideoEncodeAccelerator::Client::BitstreamBufferReady,
                 client_ptr_factory_.GetWeakPtr(),
                 bitstream_buffer.id(),
                 size,
                 key_frame));
}

}  // namespace content