1/*
2 * libjingle
3 * Copyright 2015 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 *  1. Redistributions of source code must retain the above copyright notice,
9 *     this list of conditions and the following disclaimer.
10 *  2. Redistributions in binary form must reproduce the above copyright notice,
11 *     this list of conditions and the following disclaimer in the documentation
12 *     and/or other materials provided with the distribution.
13 *  3. The name of the author may not be used to endorse or promote products
14 *     derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *
27 */
28
29#include <algorithm>
30#include <vector>
31
32#include "talk/app/webrtc/java/jni/androidmediadecoder_jni.h"
33#include "talk/app/webrtc/java/jni/androidmediacodeccommon.h"
34#include "talk/app/webrtc/java/jni/classreferenceholder.h"
35#include "talk/app/webrtc/java/jni/native_handle_impl.h"
36#include "talk/app/webrtc/java/jni/surfacetexturehelper_jni.h"
37#include "webrtc/base/bind.h"
38#include "webrtc/base/checks.h"
39#include "webrtc/base/logging.h"
40#include "webrtc/base/scoped_ref_ptr.h"
41#include "webrtc/base/thread.h"
42#include "webrtc/base/timeutils.h"
43#include "webrtc/common_video/include/i420_buffer_pool.h"
44#include "webrtc/modules/video_coding/include/video_codec_interface.h"
45#include "webrtc/system_wrappers/include/logcat_trace_context.h"
46#include "webrtc/system_wrappers/include/tick_util.h"
47#include "third_party/libyuv/include/libyuv/convert.h"
48#include "third_party/libyuv/include/libyuv/convert_from.h"
49#include "third_party/libyuv/include/libyuv/video_common.h"
50
51using rtc::Bind;
52using rtc::Thread;
53using rtc::ThreadManager;
54using rtc::scoped_ptr;
55
56using webrtc::CodecSpecificInfo;
57using webrtc::DecodedImageCallback;
58using webrtc::EncodedImage;
59using webrtc::VideoFrame;
60using webrtc::RTPFragmentationHeader;
61using webrtc::TickTime;
62using webrtc::VideoCodec;
63using webrtc::VideoCodecType;
64using webrtc::kVideoCodecH264;
65using webrtc::kVideoCodecVP8;
66using webrtc::kVideoCodecVP9;
67
68namespace webrtc_jni {
69
70class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
71                               public rtc::MessageHandler {
72 public:
73  explicit MediaCodecVideoDecoder(
74      JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context);
75  virtual ~MediaCodecVideoDecoder();
76
77  int32_t InitDecode(const VideoCodec* codecSettings, int32_t numberOfCores)
78      override;
79
80  int32_t Decode(
81      const EncodedImage& inputImage, bool missingFrames,
82      const RTPFragmentationHeader* fragmentation,
83      const CodecSpecificInfo* codecSpecificInfo = NULL,
84      int64_t renderTimeMs = -1) override;
85
86  int32_t RegisterDecodeCompleteCallback(DecodedImageCallback* callback)
87      override;
88
89  int32_t Release() override;
90
91  int32_t Reset() override;
92
93  bool PrefersLateDecoding() const override { return true; }
94
95  // rtc::MessageHandler implementation.
96  void OnMessage(rtc::Message* msg) override;
97
98  const char* ImplementationName() const override;
99
100 private:
101  // CHECK-fail if not running on |codec_thread_|.
102  void CheckOnCodecThread();
103
104  int32_t InitDecodeOnCodecThread();
105  int32_t ReleaseOnCodecThread();
106  int32_t DecodeOnCodecThread(const EncodedImage& inputImage);
107  // Deliver any outputs pending in the MediaCodec to our |callback_| and return
108  // true on success.
109  bool DeliverPendingOutputs(JNIEnv* jni, int dequeue_timeout_us);
110  int32_t ProcessHWErrorOnCodecThread();
111
112  // Type of video codec.
113  VideoCodecType codecType_;
114
115  // Render EGL context - owned by factory, should not be allocated/destroyed
116  // by VideoDecoder.
117  jobject render_egl_context_;
118
119  bool key_frame_required_;
120  bool inited_;
121  bool sw_fallback_required_;
122  bool use_surface_;
123  VideoCodec codec_;
124  webrtc::I420BufferPool decoded_frame_pool_;
125  rtc::scoped_refptr<SurfaceTextureHelper> surface_texture_helper_;
126  DecodedImageCallback* callback_;
127  int frames_received_;  // Number of frames received by decoder.
128  int frames_decoded_;  // Number of frames decoded by decoder.
129  int64_t start_time_ms_;  // Start time for statistics.
130  int current_frames_;  // Number of frames in the current statistics interval.
131  int current_bytes_;  // Encoded bytes in the current statistics interval.
132  int current_decoding_time_ms_;  // Overall decoding time in the current second
133  uint32_t max_pending_frames_;  // Maximum number of pending input frames
134
135  // State that is constant for the lifetime of this object once the ctor
136  // returns.
137  scoped_ptr<Thread> codec_thread_;  // Thread on which to operate MediaCodec.
138  ScopedGlobalRef<jclass> j_media_codec_video_decoder_class_;
139  ScopedGlobalRef<jobject> j_media_codec_video_decoder_;
140  jmethodID j_init_decode_method_;
141  jmethodID j_release_method_;
142  jmethodID j_dequeue_input_buffer_method_;
143  jmethodID j_queue_input_buffer_method_;
144  jmethodID j_dequeue_byte_buffer_method_;
145  jmethodID j_dequeue_texture_buffer_method_;
146  jmethodID j_return_decoded_byte_buffer_method_;
147  // MediaCodecVideoDecoder fields.
148  jfieldID j_input_buffers_field_;
149  jfieldID j_output_buffers_field_;
150  jfieldID j_color_format_field_;
151  jfieldID j_width_field_;
152  jfieldID j_height_field_;
153  jfieldID j_stride_field_;
154  jfieldID j_slice_height_field_;
155  // MediaCodecVideoDecoder.DecodedTextureBuffer fields.
156  jfieldID j_texture_id_field_;
157  jfieldID j_transform_matrix_field_;
158  jfieldID j_texture_timestamp_ms_field_;
159  jfieldID j_texture_ntp_timestamp_ms_field_;
160  jfieldID j_texture_decode_time_ms_field_;
161  jfieldID j_texture_frame_delay_ms_field_;
162  // MediaCodecVideoDecoder.DecodedOutputBuffer fields.
163  jfieldID j_info_index_field_;
164  jfieldID j_info_offset_field_;
165  jfieldID j_info_size_field_;
166  jfieldID j_info_timestamp_ms_field_;
167  jfieldID j_info_ntp_timestamp_ms_field_;
168  jfieldID j_byte_buffer_decode_time_ms_field_;
169
170  // Global references; must be deleted in Release().
171  std::vector<jobject> input_buffers_;
172};
173
174MediaCodecVideoDecoder::MediaCodecVideoDecoder(
175    JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context) :
176    codecType_(codecType),
177    render_egl_context_(render_egl_context),
178    key_frame_required_(true),
179    inited_(false),
180    sw_fallback_required_(false),
181    codec_thread_(new Thread()),
182    j_media_codec_video_decoder_class_(
183        jni,
184        FindClass(jni, "org/webrtc/MediaCodecVideoDecoder")),
185          j_media_codec_video_decoder_(
186              jni,
187              jni->NewObject(*j_media_codec_video_decoder_class_,
188                   GetMethodID(jni,
189                              *j_media_codec_video_decoder_class_,
190                              "<init>",
191                              "()V"))) {
192  ScopedLocalRefFrame local_ref_frame(jni);
193  codec_thread_->SetName("MediaCodecVideoDecoder", NULL);
194  RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder";
195
196  j_init_decode_method_ = GetMethodID(
197      jni, *j_media_codec_video_decoder_class_, "initDecode",
198      "(Lorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;"
199      "IILorg/webrtc/SurfaceTextureHelper;)Z");
200  j_release_method_ =
201      GetMethodID(jni, *j_media_codec_video_decoder_class_, "release", "()V");
202  j_dequeue_input_buffer_method_ = GetMethodID(
203      jni, *j_media_codec_video_decoder_class_, "dequeueInputBuffer", "()I");
204  j_queue_input_buffer_method_ = GetMethodID(
205      jni, *j_media_codec_video_decoder_class_, "queueInputBuffer", "(IIJJJ)Z");
206  j_dequeue_byte_buffer_method_ = GetMethodID(
207      jni, *j_media_codec_video_decoder_class_, "dequeueOutputBuffer",
208      "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer;");
209  j_dequeue_texture_buffer_method_ = GetMethodID(
210      jni, *j_media_codec_video_decoder_class_, "dequeueTextureBuffer",
211      "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer;");
212  j_return_decoded_byte_buffer_method_ =
213      GetMethodID(jni, *j_media_codec_video_decoder_class_,
214                  "returnDecodedOutputBuffer", "(I)V");
215
216  j_input_buffers_field_ = GetFieldID(
217      jni, *j_media_codec_video_decoder_class_,
218      "inputBuffers", "[Ljava/nio/ByteBuffer;");
219  j_output_buffers_field_ = GetFieldID(
220      jni, *j_media_codec_video_decoder_class_,
221      "outputBuffers", "[Ljava/nio/ByteBuffer;");
222  j_color_format_field_ = GetFieldID(
223      jni, *j_media_codec_video_decoder_class_, "colorFormat", "I");
224  j_width_field_ = GetFieldID(
225      jni, *j_media_codec_video_decoder_class_, "width", "I");
226  j_height_field_ = GetFieldID(
227      jni, *j_media_codec_video_decoder_class_, "height", "I");
228  j_stride_field_ = GetFieldID(
229      jni, *j_media_codec_video_decoder_class_, "stride", "I");
230  j_slice_height_field_ = GetFieldID(
231      jni, *j_media_codec_video_decoder_class_, "sliceHeight", "I");
232
233  jclass j_decoded_texture_buffer_class = FindClass(jni,
234      "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer");
235  j_texture_id_field_ = GetFieldID(
236      jni, j_decoded_texture_buffer_class, "textureID", "I");
237  j_transform_matrix_field_ = GetFieldID(
238      jni, j_decoded_texture_buffer_class, "transformMatrix", "[F");
239  j_texture_timestamp_ms_field_ = GetFieldID(
240      jni, j_decoded_texture_buffer_class, "timeStampMs", "J");
241  j_texture_ntp_timestamp_ms_field_ = GetFieldID(
242      jni, j_decoded_texture_buffer_class, "ntpTimeStampMs", "J");
243  j_texture_decode_time_ms_field_ = GetFieldID(
244      jni, j_decoded_texture_buffer_class, "decodeTimeMs", "J");
245  j_texture_frame_delay_ms_field_ = GetFieldID(
246      jni, j_decoded_texture_buffer_class, "frameDelayMs", "J");
247
248  jclass j_decoded_output_buffer_class = FindClass(jni,
249      "org/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer");
250  j_info_index_field_ = GetFieldID(
251      jni, j_decoded_output_buffer_class, "index", "I");
252  j_info_offset_field_ = GetFieldID(
253      jni, j_decoded_output_buffer_class, "offset", "I");
254  j_info_size_field_ = GetFieldID(
255      jni, j_decoded_output_buffer_class, "size", "I");
256  j_info_timestamp_ms_field_ = GetFieldID(
257      jni, j_decoded_output_buffer_class, "timeStampMs", "J");
258  j_info_ntp_timestamp_ms_field_ = GetFieldID(
259      jni, j_decoded_output_buffer_class, "ntpTimeStampMs", "J");
260  j_byte_buffer_decode_time_ms_field_ = GetFieldID(
261      jni, j_decoded_output_buffer_class, "decodeTimeMs", "J");
262
263  CHECK_EXCEPTION(jni) << "MediaCodecVideoDecoder ctor failed";
264  use_surface_ = (render_egl_context_ != NULL);
265  ALOGD << "MediaCodecVideoDecoder ctor. Use surface: " << use_surface_;
266  memset(&codec_, 0, sizeof(codec_));
267  AllowBlockingCalls();
268}
269
270MediaCodecVideoDecoder::~MediaCodecVideoDecoder() {
271  // Call Release() to ensure no more callbacks to us after we are deleted.
272  Release();
273}
274
275int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst,
276    int32_t numberOfCores) {
277  ALOGD << "InitDecode.";
278  if (inst == NULL) {
279    ALOGE << "NULL VideoCodec instance";
280    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
281  }
282  // Factory should guard against other codecs being used with us.
283  RTC_CHECK(inst->codecType == codecType_)
284      << "Unsupported codec " << inst->codecType << " for " << codecType_;
285
286  if (sw_fallback_required_) {
287    ALOGE << "InitDecode() - fallback to SW decoder";
288    return WEBRTC_VIDEO_CODEC_OK;
289  }
290  // Save VideoCodec instance for later.
291  if (&codec_ != inst) {
292    codec_ = *inst;
293  }
294  // If maxFramerate is not set then assume 30 fps.
295  codec_.maxFramerate = (codec_.maxFramerate >= 1) ? codec_.maxFramerate : 30;
296
297  // Call Java init.
298  return codec_thread_->Invoke<int32_t>(
299      Bind(&MediaCodecVideoDecoder::InitDecodeOnCodecThread, this));
300}
301
302int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
303  CheckOnCodecThread();
304  JNIEnv* jni = AttachCurrentThreadIfNeeded();
305  ScopedLocalRefFrame local_ref_frame(jni);
306  ALOGD << "InitDecodeOnCodecThread Type: " << (int)codecType_ << ". "
307      << codec_.width << " x " << codec_.height << ". Fps: " <<
308      (int)codec_.maxFramerate;
309
310  // Release previous codec first if it was allocated before.
311  int ret_val = ReleaseOnCodecThread();
312  if (ret_val < 0) {
313    ALOGE << "Release failure: " << ret_val << " - fallback to SW codec";
314    sw_fallback_required_ = true;
315    return WEBRTC_VIDEO_CODEC_ERROR;
316  }
317
318  // Always start with a complete key frame.
319  key_frame_required_ = true;
320  frames_received_ = 0;
321  frames_decoded_ = 0;
322
323  jobject java_surface_texture_helper_ = nullptr;
324  if (use_surface_) {
325    java_surface_texture_helper_ = jni->CallStaticObjectMethod(
326        FindClass(jni, "org/webrtc/SurfaceTextureHelper"),
327        GetStaticMethodID(jni,
328                          FindClass(jni, "org/webrtc/SurfaceTextureHelper"),
329                          "create",
330                          "(Lorg/webrtc/EglBase$Context;)"
331                          "Lorg/webrtc/SurfaceTextureHelper;"),
332        render_egl_context_);
333    RTC_CHECK(java_surface_texture_helper_ != nullptr);
334    surface_texture_helper_ = new rtc::RefCountedObject<SurfaceTextureHelper>(
335        jni, java_surface_texture_helper_);
336  }
337
338  jobject j_video_codec_enum = JavaEnumFromIndex(
339      jni, "MediaCodecVideoDecoder$VideoCodecType", codecType_);
340  bool success = jni->CallBooleanMethod(
341      *j_media_codec_video_decoder_,
342      j_init_decode_method_,
343      j_video_codec_enum,
344      codec_.width,
345      codec_.height,
346      java_surface_texture_helper_);
347  if (CheckException(jni) || !success) {
348    ALOGE << "Codec initialization error - fallback to SW codec.";
349    sw_fallback_required_ = true;
350    return WEBRTC_VIDEO_CODEC_ERROR;
351  }
352  inited_ = true;
353
354  switch (codecType_) {
355    case kVideoCodecVP8:
356      max_pending_frames_ = kMaxPendingFramesVp8;
357      break;
358    case kVideoCodecVP9:
359      max_pending_frames_ = kMaxPendingFramesVp9;
360      break;
361    case kVideoCodecH264:
362      max_pending_frames_ = kMaxPendingFramesH264;
363      break;
364    default:
365      max_pending_frames_ = 0;
366  }
367  start_time_ms_ = GetCurrentTimeMs();
368  current_frames_ = 0;
369  current_bytes_ = 0;
370  current_decoding_time_ms_ = 0;
371
372  jobjectArray input_buffers = (jobjectArray)GetObjectField(
373      jni, *j_media_codec_video_decoder_, j_input_buffers_field_);
374  size_t num_input_buffers = jni->GetArrayLength(input_buffers);
375  ALOGD << "Maximum amount of pending frames: " << max_pending_frames_;
376  input_buffers_.resize(num_input_buffers);
377  for (size_t i = 0; i < num_input_buffers; ++i) {
378    input_buffers_[i] =
379        jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i));
380    if (CheckException(jni)) {
381      ALOGE << "NewGlobalRef error - fallback to SW codec.";
382      sw_fallback_required_ = true;
383      return WEBRTC_VIDEO_CODEC_ERROR;
384    }
385  }
386
387  codec_thread_->PostDelayed(kMediaCodecPollMs, this);
388
389  return WEBRTC_VIDEO_CODEC_OK;
390}
391
392int32_t MediaCodecVideoDecoder::Release() {
393  ALOGD << "DecoderRelease request";
394  return codec_thread_->Invoke<int32_t>(
395        Bind(&MediaCodecVideoDecoder::ReleaseOnCodecThread, this));
396}
397
398int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() {
399  if (!inited_) {
400    return WEBRTC_VIDEO_CODEC_OK;
401  }
402  CheckOnCodecThread();
403  JNIEnv* jni = AttachCurrentThreadIfNeeded();
404  ALOGD << "DecoderReleaseOnCodecThread: Frames received: " <<
405      frames_received_ << ". Frames decoded: " << frames_decoded_;
406  ScopedLocalRefFrame local_ref_frame(jni);
407  for (size_t i = 0; i < input_buffers_.size(); i++) {
408    jni->DeleteGlobalRef(input_buffers_[i]);
409  }
410  input_buffers_.clear();
411  jni->CallVoidMethod(*j_media_codec_video_decoder_, j_release_method_);
412  surface_texture_helper_ = nullptr;
413  inited_ = false;
414  rtc::MessageQueueManager::Clear(this);
415  if (CheckException(jni)) {
416    ALOGE << "Decoder release exception";
417    return WEBRTC_VIDEO_CODEC_ERROR;
418  }
419  ALOGD << "DecoderReleaseOnCodecThread done";
420  return WEBRTC_VIDEO_CODEC_OK;
421}
422
423void MediaCodecVideoDecoder::CheckOnCodecThread() {
424  RTC_CHECK(codec_thread_ == ThreadManager::Instance()->CurrentThread())
425      << "Running on wrong thread!";
426}
427
428int32_t MediaCodecVideoDecoder::ProcessHWErrorOnCodecThread() {
429  CheckOnCodecThread();
430  int ret_val = ReleaseOnCodecThread();
431  if (ret_val < 0) {
432    ALOGE << "ProcessHWError: Release failure";
433  }
434  if (codecType_ == kVideoCodecH264) {
435    // For now there is no SW H.264 which can be used as fallback codec.
436    // So try to restart hw codec for now.
437    ret_val = InitDecodeOnCodecThread();
438    ALOGE << "Reset H.264 codec done. Status: " << ret_val;
439    if (ret_val == WEBRTC_VIDEO_CODEC_OK) {
440      // H.264 codec was succesfully reset - return regular error code.
441      return WEBRTC_VIDEO_CODEC_ERROR;
442    } else {
443      // Fail to restart H.264 codec - return error code which should stop the
444      // call.
445      return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
446    }
447  } else {
448    sw_fallback_required_ = true;
449    ALOGE << "Return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE";
450    return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
451  }
452}
453
454int32_t MediaCodecVideoDecoder::Decode(
455    const EncodedImage& inputImage,
456    bool missingFrames,
457    const RTPFragmentationHeader* fragmentation,
458    const CodecSpecificInfo* codecSpecificInfo,
459    int64_t renderTimeMs) {
460  if (sw_fallback_required_) {
461    ALOGE << "Decode() - fallback to SW codec";
462    return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
463  }
464  if (callback_ == NULL) {
465    ALOGE << "Decode() - callback_ is NULL";
466    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
467  }
468  if (inputImage._buffer == NULL && inputImage._length > 0) {
469    ALOGE << "Decode() - inputImage is incorrect";
470    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
471  }
472  if (!inited_) {
473    ALOGE << "Decode() - decoder is not initialized";
474    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
475  }
476
477  // Check if encoded frame dimension has changed.
478  if ((inputImage._encodedWidth * inputImage._encodedHeight > 0) &&
479      (inputImage._encodedWidth != codec_.width ||
480      inputImage._encodedHeight != codec_.height)) {
481    codec_.width = inputImage._encodedWidth;
482    codec_.height = inputImage._encodedHeight;
483    int32_t ret = InitDecode(&codec_, 1);
484    if (ret < 0) {
485      ALOGE << "InitDecode failure: " << ret << " - fallback to SW codec";
486      sw_fallback_required_ = true;
487      return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
488    }
489  }
490
491  // Always start with a complete key frame.
492  if (key_frame_required_) {
493    if (inputImage._frameType != webrtc::kVideoFrameKey) {
494      ALOGE << "Decode() - key frame is required";
495      return WEBRTC_VIDEO_CODEC_ERROR;
496    }
497    if (!inputImage._completeFrame) {
498      ALOGE << "Decode() - complete frame is required";
499      return WEBRTC_VIDEO_CODEC_ERROR;
500    }
501    key_frame_required_ = false;
502  }
503  if (inputImage._length == 0) {
504    return WEBRTC_VIDEO_CODEC_ERROR;
505  }
506
507  return codec_thread_->Invoke<int32_t>(Bind(
508      &MediaCodecVideoDecoder::DecodeOnCodecThread, this, inputImage));
509}
510
511int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
512    const EncodedImage& inputImage) {
513  CheckOnCodecThread();
514  JNIEnv* jni = AttachCurrentThreadIfNeeded();
515  ScopedLocalRefFrame local_ref_frame(jni);
516
517  // Try to drain the decoder and wait until output is not too
518  // much behind the input.
519  const int64 drain_start = GetCurrentTimeMs();
520  while ((frames_received_ > frames_decoded_ + max_pending_frames_) &&
521         (GetCurrentTimeMs() - drain_start) < kMediaCodecTimeoutMs) {
522    ALOGV("Received: %d. Decoded: %d. Wait for output...",
523        frames_received_, frames_decoded_);
524    if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) {
525      ALOGE << "DeliverPendingOutputs error. Frames received: " <<
526          frames_received_ << ". Frames decoded: " << frames_decoded_;
527      return ProcessHWErrorOnCodecThread();
528    }
529  }
530  if (frames_received_ > frames_decoded_ + max_pending_frames_) {
531    ALOGE << "Output buffer dequeue timeout. Frames received: " <<
532        frames_received_ << ". Frames decoded: " << frames_decoded_;
533    return ProcessHWErrorOnCodecThread();
534  }
535
536  // Get input buffer.
537  int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_decoder_,
538                                                j_dequeue_input_buffer_method_);
539  if (CheckException(jni) || j_input_buffer_index < 0) {
540    ALOGE << "dequeueInputBuffer error";
541    return ProcessHWErrorOnCodecThread();
542  }
543
544  // Copy encoded data to Java ByteBuffer.
545  jobject j_input_buffer = input_buffers_[j_input_buffer_index];
546  uint8_t* buffer =
547      reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer));
548  RTC_CHECK(buffer) << "Indirect buffer??";
549  int64_t buffer_capacity = jni->GetDirectBufferCapacity(j_input_buffer);
550  if (CheckException(jni) || buffer_capacity < inputImage._length) {
551    ALOGE << "Input frame size "<<  inputImage._length <<
552        " is bigger than buffer size " << buffer_capacity;
553    return ProcessHWErrorOnCodecThread();
554  }
555  jlong presentation_timestamp_us =
556      (frames_received_ * 1000000) / codec_.maxFramerate;
557  if (frames_decoded_ < kMaxDecodedLogFrames) {
558    ALOGD << "Decoder frame in # " << frames_received_ << ". Type: "
559        << inputImage._frameType << ". Buffer # " <<
560        j_input_buffer_index << ". pTS: "
561        << (int)(presentation_timestamp_us / 1000)
562        << ". TS: " << inputImage._timeStamp
563        << ". Size: " << inputImage._length;
564  }
565  memcpy(buffer, inputImage._buffer, inputImage._length);
566
567  // Save input image timestamps for later output.
568  frames_received_++;
569  current_bytes_ += inputImage._length;
570
571  // Feed input to decoder.
572  bool success = jni->CallBooleanMethod(
573      *j_media_codec_video_decoder_,
574      j_queue_input_buffer_method_,
575      j_input_buffer_index,
576      inputImage._length,
577      presentation_timestamp_us,
578      static_cast<int64_t> (inputImage._timeStamp),
579      inputImage.ntp_time_ms_);
580  if (CheckException(jni) || !success) {
581    ALOGE << "queueInputBuffer error";
582    return ProcessHWErrorOnCodecThread();
583  }
584
585  // Try to drain the decoder
586  if (!DeliverPendingOutputs(jni, 0)) {
587    ALOGE << "DeliverPendingOutputs error";
588    return ProcessHWErrorOnCodecThread();
589  }
590
591  return WEBRTC_VIDEO_CODEC_OK;
592}
593
594bool MediaCodecVideoDecoder::DeliverPendingOutputs(
595    JNIEnv* jni, int dequeue_timeout_ms) {
596  if (frames_received_ <= frames_decoded_) {
597    // No need to query for output buffers - decoder is drained.
598    return true;
599  }
600  // Get decoder output.
601  jobject j_decoder_output_buffer =
602      jni->CallObjectMethod(*j_media_codec_video_decoder_,
603          use_surface_ ? j_dequeue_texture_buffer_method_
604                       : j_dequeue_byte_buffer_method_,
605          dequeue_timeout_ms);
606
607  if (CheckException(jni)) {
608    ALOGE << "dequeueOutputBuffer() error";
609    return false;
610  }
611  if (IsNull(jni, j_decoder_output_buffer)) {
612    // No decoded frame ready.
613    return true;
614  }
615
616  // Get decoded video frame properties.
617  int color_format = GetIntField(jni, *j_media_codec_video_decoder_,
618      j_color_format_field_);
619  int width = GetIntField(jni, *j_media_codec_video_decoder_, j_width_field_);
620  int height = GetIntField(jni, *j_media_codec_video_decoder_, j_height_field_);
621  int stride = GetIntField(jni, *j_media_codec_video_decoder_, j_stride_field_);
622  int slice_height = GetIntField(jni, *j_media_codec_video_decoder_,
623      j_slice_height_field_);
624
625  rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer;
626  int64_t output_timestamps_ms = 0;
627  int64_t output_ntp_timestamps_ms = 0;
628  int decode_time_ms = 0;
629  int64_t frame_delayed_ms = 0;
630  if (use_surface_) {
631    // Extract data from Java DecodedTextureBuffer.
632    const int texture_id =
633        GetIntField(jni, j_decoder_output_buffer, j_texture_id_field_);
634    if (texture_id != 0) {  // |texture_id| == 0 represents a dropped frame.
635      const jfloatArray j_transform_matrix =
636          reinterpret_cast<jfloatArray>(GetObjectField(
637              jni, j_decoder_output_buffer, j_transform_matrix_field_));
638      const int64_t timestamp_us =
639          GetLongField(jni, j_decoder_output_buffer,
640              j_texture_timestamp_ms_field_);
641      output_timestamps_ms = GetLongField(jni, j_decoder_output_buffer,
642                                          j_texture_timestamp_ms_field_);
643      output_ntp_timestamps_ms =
644          GetLongField(jni, j_decoder_output_buffer,
645                       j_texture_ntp_timestamp_ms_field_);
646      decode_time_ms = GetLongField(jni, j_decoder_output_buffer,
647          j_texture_decode_time_ms_field_);
648      frame_delayed_ms = GetLongField(jni, j_decoder_output_buffer,
649          j_texture_frame_delay_ms_field_);
650
651      // Create webrtc::VideoFrameBuffer with native texture handle.
652      frame_buffer = surface_texture_helper_->CreateTextureFrame(
653          width, height, NativeHandleImpl(jni, texture_id, j_transform_matrix));
654    }
655  } else {
656    // Extract data from Java ByteBuffer and create output yuv420 frame -
657    // for non surface decoding only.
658    const int output_buffer_index =
659        GetIntField(jni, j_decoder_output_buffer, j_info_index_field_);
660    const int output_buffer_offset =
661        GetIntField(jni, j_decoder_output_buffer, j_info_offset_field_);
662    const int output_buffer_size =
663        GetIntField(jni, j_decoder_output_buffer, j_info_size_field_);
664    output_timestamps_ms = GetLongField(jni, j_decoder_output_buffer,
665                                        j_info_timestamp_ms_field_);
666    output_ntp_timestamps_ms =
667        GetLongField(jni, j_decoder_output_buffer,
668                     j_info_ntp_timestamp_ms_field_);
669
670    decode_time_ms = GetLongField(jni, j_decoder_output_buffer,
671                                  j_byte_buffer_decode_time_ms_field_);
672
673    if (output_buffer_size < width * height * 3 / 2) {
674      ALOGE << "Insufficient output buffer size: " << output_buffer_size;
675      return false;
676    }
677    jobjectArray output_buffers = reinterpret_cast<jobjectArray>(GetObjectField(
678        jni, *j_media_codec_video_decoder_, j_output_buffers_field_));
679    jobject output_buffer =
680        jni->GetObjectArrayElement(output_buffers, output_buffer_index);
681    uint8_t* payload = reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(
682        output_buffer));
683    if (CheckException(jni)) {
684      return false;
685    }
686    payload += output_buffer_offset;
687
688    // Create yuv420 frame.
689    frame_buffer = decoded_frame_pool_.CreateBuffer(width, height);
690    if (color_format == COLOR_FormatYUV420Planar) {
691      RTC_CHECK_EQ(0, stride % 2);
692      RTC_CHECK_EQ(0, slice_height % 2);
693      const int uv_stride = stride / 2;
694      const int u_slice_height = slice_height / 2;
695      const uint8_t* y_ptr = payload;
696      const uint8_t* u_ptr = y_ptr + stride * slice_height;
697      const uint8_t* v_ptr = u_ptr + uv_stride * u_slice_height;
698      libyuv::I420Copy(y_ptr, stride,
699                       u_ptr, uv_stride,
700                       v_ptr, uv_stride,
701                       frame_buffer->MutableData(webrtc::kYPlane),
702                       frame_buffer->stride(webrtc::kYPlane),
703                       frame_buffer->MutableData(webrtc::kUPlane),
704                       frame_buffer->stride(webrtc::kUPlane),
705                       frame_buffer->MutableData(webrtc::kVPlane),
706                       frame_buffer->stride(webrtc::kVPlane),
707                       width, height);
708    } else {
709      // All other supported formats are nv12.
710      const uint8_t* y_ptr = payload;
711      const uint8_t* uv_ptr = y_ptr + stride * slice_height;
712      libyuv::NV12ToI420(
713          y_ptr, stride,
714          uv_ptr, stride,
715          frame_buffer->MutableData(webrtc::kYPlane),
716          frame_buffer->stride(webrtc::kYPlane),
717          frame_buffer->MutableData(webrtc::kUPlane),
718          frame_buffer->stride(webrtc::kUPlane),
719          frame_buffer->MutableData(webrtc::kVPlane),
720          frame_buffer->stride(webrtc::kVPlane),
721          width, height);
722    }
723    // Return output byte buffer back to codec.
724    jni->CallVoidMethod(
725        *j_media_codec_video_decoder_,
726        j_return_decoded_byte_buffer_method_,
727        output_buffer_index);
728    if (CheckException(jni)) {
729      ALOGE << "returnDecodedOutputBuffer error";
730      return false;
731    }
732  }
733  VideoFrame decoded_frame(frame_buffer, 0, 0, webrtc::kVideoRotation_0);
734  decoded_frame.set_timestamp(output_timestamps_ms);
735  decoded_frame.set_ntp_time_ms(output_ntp_timestamps_ms);
736
737  if (frames_decoded_ < kMaxDecodedLogFrames) {
738    ALOGD << "Decoder frame out # " << frames_decoded_ << ". " << width <<
739        " x " << height << ". " << stride << " x " <<  slice_height <<
740        ". Color: " << color_format << ". TS:" << decoded_frame.timestamp() <<
741        ". DecTime: " << (int)decode_time_ms <<
742        ". DelayTime: " << (int)frame_delayed_ms;
743  }
744
745  // Calculate and print decoding statistics - every 3 seconds.
746  frames_decoded_++;
747  current_frames_++;
748  current_decoding_time_ms_ += decode_time_ms;
749  int statistic_time_ms = GetCurrentTimeMs() - start_time_ms_;
750  if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs &&
751      current_frames_ > 0) {
752    ALOGD << "Decoded frames: " << frames_decoded_ <<  ". Received frames: "
753        <<  frames_received_ << ".  Bitrate: " <<
754        (current_bytes_ * 8 / statistic_time_ms) << " kbps, fps: " <<
755        ((current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms)
756        << ". decTime: " << (current_decoding_time_ms_ / current_frames_) <<
757        " for last " << statistic_time_ms << " ms.";
758    start_time_ms_ = GetCurrentTimeMs();
759    current_frames_ = 0;
760    current_bytes_ = 0;
761    current_decoding_time_ms_ = 0;
762  }
763
764  // |.IsZeroSize())| returns true when a frame has been dropped.
765  if (!decoded_frame.IsZeroSize()) {
766    // Callback - output decoded frame.
767    const int32_t callback_status =
768        callback_->Decoded(decoded_frame, decode_time_ms);
769    if (callback_status > 0) {
770      ALOGE << "callback error";
771    }
772  }
773  return true;
774}
775
776int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback(
777    DecodedImageCallback* callback) {
778  callback_ = callback;
779  return WEBRTC_VIDEO_CODEC_OK;
780}
781
782int32_t MediaCodecVideoDecoder::Reset() {
783  ALOGD << "DecoderReset";
784  if (!inited_) {
785    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
786  }
787  return InitDecode(&codec_, 1);
788}
789
790void MediaCodecVideoDecoder::OnMessage(rtc::Message* msg) {
791  JNIEnv* jni = AttachCurrentThreadIfNeeded();
792  ScopedLocalRefFrame local_ref_frame(jni);
793  if (!inited_) {
794    return;
795  }
796  // We only ever send one message to |this| directly (not through a Bind()'d
797  // functor), so expect no ID/data.
798  RTC_CHECK(!msg->message_id) << "Unexpected message!";
799  RTC_CHECK(!msg->pdata) << "Unexpected message!";
800  CheckOnCodecThread();
801
802  if (!DeliverPendingOutputs(jni, 0)) {
803    ALOGE << "OnMessage: DeliverPendingOutputs error";
804    ProcessHWErrorOnCodecThread();
805    return;
806  }
807  codec_thread_->PostDelayed(kMediaCodecPollMs, this);
808}
809
810MediaCodecVideoDecoderFactory::MediaCodecVideoDecoderFactory() :
811    render_egl_context_(NULL) {
812  ALOGD << "MediaCodecVideoDecoderFactory ctor";
813  JNIEnv* jni = AttachCurrentThreadIfNeeded();
814  ScopedLocalRefFrame local_ref_frame(jni);
815  jclass j_decoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoDecoder");
816  supported_codec_types_.clear();
817
818  bool is_vp8_hw_supported = jni->CallStaticBooleanMethod(
819      j_decoder_class,
820      GetStaticMethodID(jni, j_decoder_class, "isVp8HwSupported", "()Z"));
821  if (CheckException(jni)) {
822    is_vp8_hw_supported = false;
823  }
824  if (is_vp8_hw_supported) {
825    ALOGD << "VP8 HW Decoder supported.";
826    supported_codec_types_.push_back(kVideoCodecVP8);
827  }
828
829  bool is_vp9_hw_supported = jni->CallStaticBooleanMethod(
830      j_decoder_class,
831      GetStaticMethodID(jni, j_decoder_class, "isVp9HwSupported", "()Z"));
832  if (CheckException(jni)) {
833    is_vp9_hw_supported = false;
834  }
835  if (is_vp9_hw_supported) {
836    ALOGD << "VP9 HW Decoder supported.";
837    supported_codec_types_.push_back(kVideoCodecVP9);
838  }
839
840  bool is_h264_hw_supported = jni->CallStaticBooleanMethod(
841      j_decoder_class,
842      GetStaticMethodID(jni, j_decoder_class, "isH264HwSupported", "()Z"));
843  if (CheckException(jni)) {
844    is_h264_hw_supported = false;
845  }
846  if (is_h264_hw_supported) {
847    ALOGD << "H264 HW Decoder supported.";
848    supported_codec_types_.push_back(kVideoCodecH264);
849  }
850}
851
852MediaCodecVideoDecoderFactory::~MediaCodecVideoDecoderFactory() {
853  ALOGD << "MediaCodecVideoDecoderFactory dtor";
854  if (render_egl_context_) {
855    JNIEnv* jni = AttachCurrentThreadIfNeeded();
856    jni->DeleteGlobalRef(render_egl_context_);
857    render_egl_context_ = NULL;
858  }
859}
860
861void MediaCodecVideoDecoderFactory::SetEGLContext(
862    JNIEnv* jni, jobject render_egl_context) {
863  ALOGD << "MediaCodecVideoDecoderFactory::SetEGLContext";
864  if (render_egl_context_) {
865    jni->DeleteGlobalRef(render_egl_context_);
866    render_egl_context_ = NULL;
867  }
868  if (!IsNull(jni, render_egl_context)) {
869    render_egl_context_ = jni->NewGlobalRef(render_egl_context);
870    if (CheckException(jni)) {
871      ALOGE << "error calling NewGlobalRef for EGL Context.";
872      render_egl_context_ = NULL;
873    } else {
874      jclass j_egl_context_class =
875          FindClass(jni, "org/webrtc/EglBase$Context");
876      if (!jni->IsInstanceOf(render_egl_context_, j_egl_context_class)) {
877        ALOGE << "Wrong EGL Context.";
878        jni->DeleteGlobalRef(render_egl_context_);
879        render_egl_context_ = NULL;
880      }
881    }
882  }
883  if (render_egl_context_ == NULL) {
884    ALOGW << "NULL VideoDecoder EGL context - HW surface decoding is disabled.";
885  }
886}
887
888webrtc::VideoDecoder* MediaCodecVideoDecoderFactory::CreateVideoDecoder(
889    VideoCodecType type) {
890  if (supported_codec_types_.empty()) {
891    ALOGW << "No HW video decoder for type " << (int)type;
892    return NULL;
893  }
894  for (VideoCodecType codec_type : supported_codec_types_) {
895    if (codec_type == type) {
896      ALOGD << "Create HW video decoder for type " << (int)type;
897      return new MediaCodecVideoDecoder(
898          AttachCurrentThreadIfNeeded(), type, render_egl_context_);
899    }
900  }
901  ALOGW << "Can not find HW video decoder for type " << (int)type;
902  return NULL;
903}
904
905void MediaCodecVideoDecoderFactory::DestroyVideoDecoder(
906    webrtc::VideoDecoder* decoder) {
907  ALOGD << "Destroy video decoder.";
908  delete decoder;
909}
910
911const char* MediaCodecVideoDecoder::ImplementationName() const {
912  return "MediaCodec";
913}
914
915}  // namespace webrtc_jni
916
917