media_source_player.cc revision 3551c9c881056c480085172ff9840cab31610854 
1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "media/base/android/media_source_player.h"
6
7#include "base/android/jni_android.h"
8#include "base/android/jni_string.h"
9#include "base/basictypes.h"
10#include "base/bind.h"
11#include "base/logging.h"
12#include "media/base/android/audio_decoder_job.h"
13#include "media/base/android/media_drm_bridge.h"
14#include "media/base/android/media_player_manager.h"
15#include "media/base/android/video_decoder_job.h"
16#include "media/base/audio_timestamp_helper.h"
17
18namespace {
19
20// Use 16bit PCM for audio output. Keep this value in sync with the output
21// format we passed to AudioTrack in MediaCodecBridge.
22const int kBytesPerAudioOutputSample = 2;
23}
24
25namespace media {
26
27MediaSourcePlayer::MediaSourcePlayer(
28    int player_id,
29    MediaPlayerManager* manager)
30    : MediaPlayerAndroid(player_id, manager),
31      pending_event_(NO_EVENT_PENDING),
32      seek_request_id_(0),
33      width_(0),
34      height_(0),
35      audio_codec_(kUnknownAudioCodec),
36      video_codec_(kUnknownVideoCodec),
37      num_channels_(0),
38      sampling_rate_(0),
39      audio_finished_(true),
40      video_finished_(true),
41      playing_(false),
42      is_audio_encrypted_(false),
43      is_video_encrypted_(false),
44      volume_(-1.0),
45      clock_(&default_tick_clock_),
46      reconfig_audio_decoder_(false),
47      reconfig_video_decoder_(false),
48      audio_access_unit_index_(0),
49      video_access_unit_index_(0),
50      waiting_for_audio_data_(false),
51      waiting_for_video_data_(false),
52      sync_decoder_jobs_(true),
53      weak_this_(this),
54      drm_bridge_(NULL) {
55}
56
57MediaSourcePlayer::~MediaSourcePlayer() {
58  Release();
59}
60
61void MediaSourcePlayer::SetVideoSurface(gfx::ScopedJavaSurface surface) {
62  // Ignore non-empty surface that is unprotected if |is_video_encrypted_| is
63  // true.
64  if (is_video_encrypted_ && !surface.IsEmpty() && !surface.is_protected())
65    return;
66
67  surface_ =  surface.Pass();
68  pending_event_ |= SURFACE_CHANGE_EVENT_PENDING;
69  if (pending_event_ & SEEK_EVENT_PENDING) {
70    // Waiting for the seek to finish.
71    return;
72  }
73  // Setting a new surface will require a new MediaCodec to be created.
74  // Request a seek so that the new decoder will decode an I-frame first.
75  // Or otherwise, the new MediaCodec might crash. See b/8950387.
76  pending_event_ |= SEEK_EVENT_PENDING;
77  ProcessPendingEvents();
78}
79
80bool MediaSourcePlayer::Seekable() {
81  // If the duration TimeDelta, converted to milliseconds from microseconds,
82  // is >= 2^31, then the media is assumed to be unbounded and unseekable.
83  // 2^31 is the bound due to java player using 32-bit integer for time
84  // values at millisecond resolution.
85  return duration_ <
86         base::TimeDelta::FromMilliseconds(std::numeric_limits<int32>::max());
87}
88
89void MediaSourcePlayer::Start() {
90  playing_ = true;
91
92  if (is_video_encrypted_)
93    manager()->OnProtectedSurfaceRequested(player_id());
94
95  StartInternal();
96}
97
98void MediaSourcePlayer::Pause() {
99  // Since decoder jobs have their own thread, decoding is not fully paused
100  // until all the decoder jobs call MediaDecoderCallback(). It is possible
101  // that Start() is called while the player is waiting for
102  // MediaDecoderCallback(). In that case, decoding will continue when
103  // MediaDecoderCallback() is called.
104  playing_ = false;
105  start_time_ticks_ = base::TimeTicks();
106}
107
108bool MediaSourcePlayer::IsPlaying() {
109  return playing_;
110}
111
112int MediaSourcePlayer::GetVideoWidth() {
113  return width_;
114}
115
116int MediaSourcePlayer::GetVideoHeight() {
117  return height_;
118}
119
120void MediaSourcePlayer::SeekTo(base::TimeDelta timestamp) {
121  clock_.SetTime(timestamp, timestamp);
122  if (audio_timestamp_helper_)
123    audio_timestamp_helper_->SetBaseTimestamp(timestamp);
124  pending_event_ |= SEEK_EVENT_PENDING;
125  ProcessPendingEvents();
126}
127
128base::TimeDelta MediaSourcePlayer::GetCurrentTime() {
129  return clock_.Elapsed();
130}
131
132base::TimeDelta MediaSourcePlayer::GetDuration() {
133  return duration_;
134}
135
136void MediaSourcePlayer::Release() {
137  ClearDecodingData();
138  audio_decoder_job_.reset();
139  video_decoder_job_.reset();
140  reconfig_audio_decoder_ = false;
141  reconfig_video_decoder_ = false;
142  playing_ = false;
143  pending_event_ = NO_EVENT_PENDING;
144  surface_ = gfx::ScopedJavaSurface();
145  ReleaseMediaResourcesFromManager();
146}
147
148void MediaSourcePlayer::SetVolume(double volume) {
149  volume_ = volume;
150  SetVolumeInternal();
151}
152
153bool MediaSourcePlayer::CanPause() {
154  return Seekable();
155}
156
157bool MediaSourcePlayer::CanSeekForward() {
158  return Seekable();
159}
160
161bool MediaSourcePlayer::CanSeekBackward() {
162  return Seekable();
163}
164
165bool MediaSourcePlayer::IsPlayerReady() {
166  return audio_decoder_job_ || video_decoder_job_;
167}
168
169void MediaSourcePlayer::StartInternal() {
170  // If there are pending events, wait for them finish.
171  if (pending_event_ != NO_EVENT_PENDING)
172    return;
173
174  // Create decoder jobs if they are not created
175  ConfigureAudioDecoderJob();
176  ConfigureVideoDecoderJob();
177
178
179  // If one of the decoder job is not ready, do nothing.
180  if ((HasAudio() && !audio_decoder_job_) ||
181      (HasVideo() && !video_decoder_job_)) {
182    return;
183  }
184
185  audio_finished_ = false;
186  video_finished_ = false;
187  sync_decoder_jobs_ = true;
188  SyncAndStartDecoderJobs();
189}
190
191void MediaSourcePlayer::DemuxerReady(
192    const MediaPlayerHostMsg_DemuxerReady_Params& params) {
193  duration_ = base::TimeDelta::FromMilliseconds(params.duration_ms);
194  clock_.SetDuration(duration_);
195
196  audio_codec_ = params.audio_codec;
197  num_channels_ = params.audio_channels;
198  sampling_rate_ = params.audio_sampling_rate;
199  is_audio_encrypted_ = params.is_audio_encrypted;
200  audio_extra_data_ = params.audio_extra_data;
201  if (HasAudio()) {
202    DCHECK_GT(num_channels_, 0);
203    audio_timestamp_helper_.reset(new AudioTimestampHelper(sampling_rate_));
204    audio_timestamp_helper_->SetBaseTimestamp(GetCurrentTime());
205  } else {
206    audio_timestamp_helper_.reset();
207  }
208
209  video_codec_ = params.video_codec;
210  width_ = params.video_size.width();
211  height_ = params.video_size.height();
212  is_video_encrypted_ = params.is_video_encrypted;
213
214  OnMediaMetadataChanged(duration_, width_, height_, true);
215
216  if (pending_event_ & CONFIG_CHANGE_EVENT_PENDING) {
217    if (reconfig_audio_decoder_)
218      ConfigureAudioDecoderJob();
219
220    // If there is a pending surface change, we can merge it with the config
221    // change.
222    if (reconfig_video_decoder_) {
223      pending_event_ &= ~SURFACE_CHANGE_EVENT_PENDING;
224      ConfigureVideoDecoderJob();
225    }
226    pending_event_ &= ~CONFIG_CHANGE_EVENT_PENDING;
227    if (playing_)
228      StartInternal();
229  }
230}
231
232void MediaSourcePlayer::ReadFromDemuxerAck(
233    const MediaPlayerHostMsg_ReadFromDemuxerAck_Params& params) {
234  DCHECK_LT(0u, params.access_units.size());
235  if (params.type == DemuxerStream::AUDIO)
236    waiting_for_audio_data_ = false;
237  else
238    waiting_for_video_data_ = false;
239
240  // If there is a pending seek request, ignore the data from the chunk demuxer.
241  // The data will be requested later when OnSeekRequestAck() is called.
242  if (pending_event_ & SEEK_EVENT_PENDING)
243    return;
244
245  if (params.type == DemuxerStream::AUDIO) {
246    DCHECK_EQ(0u, audio_access_unit_index_);
247    received_audio_ = params;
248  } else {
249    DCHECK_EQ(0u, video_access_unit_index_);
250    received_video_ = params;
251  }
252
253  if (pending_event_ != NO_EVENT_PENDING || !playing_)
254    return;
255
256  if (sync_decoder_jobs_) {
257    SyncAndStartDecoderJobs();
258    return;
259  }
260
261  if (params.type == DemuxerStream::AUDIO)
262    DecodeMoreAudio();
263  else
264    DecodeMoreVideo();
265}
266
267void MediaSourcePlayer::DurationChanged(const base::TimeDelta& duration) {
268  duration_ = duration;
269  clock_.SetDuration(duration_);
270}
271
272void MediaSourcePlayer::SetDrmBridge(MediaDrmBridge* drm_bridge) {
273  // Currently we don't support DRM change during the middle of playback, even
274  // if the player is paused.
275  // TODO(qinmin): support DRM change after playback has started.
276  // http://crbug.com/253792.
277  if (GetCurrentTime() > base::TimeDelta()) {
278    LOG(INFO) << "Setting DRM bridge after play back has started. "
279              << "This is not well supported!";
280  }
281
282  drm_bridge_ = drm_bridge;
283
284  if (playing_)
285    StartInternal();
286}
287
288void MediaSourcePlayer::OnSeekRequestAck(unsigned seek_request_id) {
289  DVLOG(1) << "OnSeekRequestAck(" << seek_request_id << ")";
290  // Do nothing until the most recent seek request is processed.
291  if (seek_request_id_ != seek_request_id)
292    return;
293  pending_event_ &= ~SEEK_EVENT_PENDING;
294  OnSeekComplete();
295  ProcessPendingEvents();
296}
297
298void MediaSourcePlayer::UpdateTimestamps(
299    const base::TimeDelta& presentation_timestamp, size_t audio_output_bytes) {
300  if (audio_output_bytes > 0) {
301    audio_timestamp_helper_->AddFrames(
302        audio_output_bytes / (kBytesPerAudioOutputSample * num_channels_));
303    clock_.SetMaxTime(audio_timestamp_helper_->GetTimestamp());
304  } else {
305    clock_.SetMaxTime(presentation_timestamp);
306  }
307
308  OnTimeUpdated();
309}
310
311void MediaSourcePlayer::ProcessPendingEvents() {
312  // Wait for all the decoding jobs to finish before processing pending tasks.
313  if ((audio_decoder_job_ && audio_decoder_job_->is_decoding()) ||
314      (video_decoder_job_ && video_decoder_job_->is_decoding())) {
315    return;
316  }
317
318  if (pending_event_ & SEEK_EVENT_PENDING) {
319    ClearDecodingData();
320    manager()->OnMediaSeekRequest(
321        player_id(), GetCurrentTime(), ++seek_request_id_);
322    return;
323  }
324
325  start_time_ticks_ = base::TimeTicks();
326  if (pending_event_ & CONFIG_CHANGE_EVENT_PENDING) {
327    DCHECK(reconfig_audio_decoder_ || reconfig_video_decoder_);
328    manager()->OnMediaConfigRequest(player_id());
329    return;
330  }
331
332  if (pending_event_ & SURFACE_CHANGE_EVENT_PENDING) {
333    video_decoder_job_.reset();
334    ConfigureVideoDecoderJob();
335    pending_event_ &= ~SURFACE_CHANGE_EVENT_PENDING;
336  }
337
338  if (playing_)
339    StartInternal();
340}
341
342void MediaSourcePlayer::MediaDecoderCallback(
343    bool is_audio, MediaDecoderJob::DecodeStatus decode_status,
344    const base::TimeDelta& presentation_timestamp, size_t audio_output_bytes) {
345  if (is_audio && audio_decoder_job_)
346    audio_decoder_job_->OnDecodeCompleted();
347  if (!is_audio && video_decoder_job_)
348    video_decoder_job_->OnDecodeCompleted();
349
350  if (is_audio)
351    decoder_starvation_callback_.Cancel();
352
353  if (decode_status == MediaDecoderJob::DECODE_FAILED) {
354    Release();
355    OnMediaError(MEDIA_ERROR_DECODE);
356    return;
357  }
358
359  // If the input reaches input EOS, there is no need to request new data.
360  if (decode_status != MediaDecoderJob::DECODE_TRY_ENQUEUE_INPUT_AGAIN_LATER &&
361      decode_status != MediaDecoderJob::DECODE_INPUT_END_OF_STREAM) {
362    if (is_audio)
363      audio_access_unit_index_++;
364    else
365      video_access_unit_index_++;
366  }
367
368  if (pending_event_ != NO_EVENT_PENDING) {
369    ProcessPendingEvents();
370    return;
371  }
372
373  if (decode_status == MediaDecoderJob::DECODE_SUCCEEDED &&
374      (is_audio || !HasAudio())) {
375    UpdateTimestamps(presentation_timestamp, audio_output_bytes);
376  }
377
378  if (decode_status == MediaDecoderJob::DECODE_OUTPUT_END_OF_STREAM) {
379    PlaybackCompleted(is_audio);
380    return;
381  }
382
383  if (!playing_) {
384    if (is_audio || !HasAudio())
385      clock_.Pause();
386    return;
387  }
388
389  if (sync_decoder_jobs_) {
390    SyncAndStartDecoderJobs();
391    return;
392  }
393
394  base::TimeDelta current_timestamp = GetCurrentTime();
395  if (is_audio) {
396    if (decode_status == MediaDecoderJob::DECODE_SUCCEEDED) {
397      base::TimeDelta timeout =
398          audio_timestamp_helper_->GetTimestamp() - current_timestamp;
399      StartStarvationCallback(timeout);
400    }
401    if (!HasAudioData())
402      RequestAudioData();
403    else
404      DecodeMoreAudio();
405    return;
406  }
407
408  if (!HasAudio() && decode_status == MediaDecoderJob::DECODE_SUCCEEDED) {
409    DCHECK(current_timestamp <= presentation_timestamp);
410    // For video only streams, fps can be estimated from the difference
411    // between the previous and current presentation timestamps. The
412    // previous presentation timestamp is equal to current_timestamp.
413    // TODO(qinmin): determine whether 2 is a good coefficient for estimating
414    // video frame timeout.
415    StartStarvationCallback(2 * (presentation_timestamp - current_timestamp));
416  }
417  if (!HasVideoData())
418    RequestVideoData();
419  else
420    DecodeMoreVideo();
421}
422
423void MediaSourcePlayer::DecodeMoreAudio() {
424  DCHECK(!audio_decoder_job_->is_decoding());
425  DCHECK(HasAudioData());
426
427  if (DemuxerStream::kConfigChanged ==
428      received_audio_.access_units[audio_access_unit_index_].status) {
429    // Wait for demuxer ready message.
430    reconfig_audio_decoder_ = true;
431    pending_event_ |= CONFIG_CHANGE_EVENT_PENDING;
432    received_audio_ = MediaPlayerHostMsg_ReadFromDemuxerAck_Params();
433    audio_access_unit_index_ = 0;
434    ProcessPendingEvents();
435    return;
436  }
437
438  audio_decoder_job_->Decode(
439      received_audio_.access_units[audio_access_unit_index_],
440      start_time_ticks_, start_presentation_timestamp_,
441      base::Bind(&MediaSourcePlayer::MediaDecoderCallback,
442                 weak_this_.GetWeakPtr(), true));
443}
444
445void MediaSourcePlayer::DecodeMoreVideo() {
446  DVLOG(1) << "DecodeMoreVideo()";
447  DCHECK(!video_decoder_job_->is_decoding());
448  DCHECK(HasVideoData());
449
450  if (DemuxerStream::kConfigChanged ==
451      received_video_.access_units[video_access_unit_index_].status) {
452    // Wait for demuxer ready message.
453    reconfig_video_decoder_ = true;
454    pending_event_ |= CONFIG_CHANGE_EVENT_PENDING;
455    received_video_ = MediaPlayerHostMsg_ReadFromDemuxerAck_Params();
456    video_access_unit_index_ = 0;
457    ProcessPendingEvents();
458    return;
459  }
460
461  DVLOG(3) << "VideoDecoderJob::Decode(" << video_access_unit_index_ << ", "
462           << start_time_ticks_.ToInternalValue() << ", "
463           << start_presentation_timestamp_.InMilliseconds() << ")";
464  video_decoder_job_->Decode(
465      received_video_.access_units[video_access_unit_index_],
466      start_time_ticks_, start_presentation_timestamp_,
467      base::Bind(&MediaSourcePlayer::MediaDecoderCallback,
468                 weak_this_.GetWeakPtr(), false));
469}
470
471void MediaSourcePlayer::PlaybackCompleted(bool is_audio) {
472  if (is_audio)
473    audio_finished_ = true;
474  else
475    video_finished_ = true;
476
477  if ((!HasAudio() || audio_finished_) && (!HasVideo() || video_finished_)) {
478    playing_ = false;
479    clock_.Pause();
480    start_time_ticks_ = base::TimeTicks();
481    OnPlaybackComplete();
482  }
483}
484
485void MediaSourcePlayer::ClearDecodingData() {
486  DVLOG(1) << "ClearDecodingData()";
487  if (audio_decoder_job_)
488    audio_decoder_job_->Flush();
489  if (video_decoder_job_)
490    video_decoder_job_->Flush();
491  start_time_ticks_ = base::TimeTicks();
492  received_audio_ = MediaPlayerHostMsg_ReadFromDemuxerAck_Params();
493  received_video_ = MediaPlayerHostMsg_ReadFromDemuxerAck_Params();
494  audio_access_unit_index_ = 0;
495  video_access_unit_index_ = 0;
496  waiting_for_audio_data_ = false;
497  waiting_for_video_data_ = false;
498}
499
500bool MediaSourcePlayer::HasVideo() {
501  return kUnknownVideoCodec != video_codec_;
502}
503
504bool MediaSourcePlayer::HasAudio() {
505  return kUnknownAudioCodec != audio_codec_;
506}
507
508void MediaSourcePlayer::ConfigureAudioDecoderJob() {
509  if (!HasAudio()) {
510    audio_decoder_job_.reset();
511    return;
512  }
513
514  // Create audio decoder job only if config changes.
515  if (audio_decoder_job_ && !reconfig_audio_decoder_)
516    return;
517
518  base::android::ScopedJavaLocalRef<jobject> media_codec;
519  if (is_audio_encrypted_) {
520    if (drm_bridge_) {
521      media_codec = drm_bridge_->GetMediaCrypto();
522      // TODO(qinmin): currently we assume MediaCrypto is available whenever
523      // MediaDrmBridge is constructed. This will change if we want to support
524      // more general uses cases of EME.
525      DCHECK(!media_codec.is_null());
526    } else {
527      // Don't create the decoder job if |drm_bridge_| is not set,
528      // so StartInternal() will not proceed.
529      LOG(INFO) << "MediaDrmBridge is not available when creating decoder "
530                << "for encrypted audio stream.";
531      return;
532    }
533  }
534
535  audio_decoder_job_.reset(AudioDecoderJob::Create(
536      audio_codec_, sampling_rate_, num_channels_, &audio_extra_data_[0],
537      audio_extra_data_.size(), media_codec.obj()));
538
539  if (audio_decoder_job_) {
540    SetVolumeInternal();
541    reconfig_audio_decoder_ =  false;
542  }
543}
544
545void MediaSourcePlayer::ConfigureVideoDecoderJob() {
546  if (!HasVideo() || surface_.IsEmpty()) {
547    video_decoder_job_.reset();
548    return;
549  }
550
551  // Create video decoder job only if config changes.
552  if (video_decoder_job_ && !reconfig_video_decoder_)
553    return;
554
555  base::android::ScopedJavaLocalRef<jobject> media_crypto;
556  if (is_video_encrypted_) {
557    if (drm_bridge_) {
558      media_crypto = drm_bridge_->GetMediaCrypto();
559      DCHECK(!media_crypto.is_null());
560    } else {
561      LOG(INFO) << "MediaDrmBridge is not available when creating decoder "
562                << "for encrypted video stream.";
563      return;
564    }
565  }
566
567  // Release the old VideoDecoderJob first so the surface can get released.
568  // Android does not allow 2 MediaCodec instances use the same surface.
569  video_decoder_job_.reset();
570  // Create the new VideoDecoderJob.
571  video_decoder_job_.reset(VideoDecoderJob::Create(
572      video_codec_, gfx::Size(width_, height_), surface_.j_surface().obj(),
573      media_crypto.obj()));
574  if (video_decoder_job_)
575    reconfig_video_decoder_ = false;
576
577  // Inform the fullscreen view the player is ready.
578  // TODO(qinmin): refactor MediaPlayerBridge so that we have a better way
579  // to inform ContentVideoView.
580  OnMediaMetadataChanged(duration_, width_, height_, true);
581}
582
583void MediaSourcePlayer::OnDecoderStarved() {
584  sync_decoder_jobs_ = true;
585}
586
587void MediaSourcePlayer::StartStarvationCallback(
588    const base::TimeDelta& timeout) {
589  decoder_starvation_callback_.Reset(
590      base::Bind(&MediaSourcePlayer::OnDecoderStarved,
591                 weak_this_.GetWeakPtr()));
592  base::MessageLoop::current()->PostDelayedTask(
593      FROM_HERE, decoder_starvation_callback_.callback(), timeout);
594}
595
596void MediaSourcePlayer::SyncAndStartDecoderJobs() {
597  // For streams with both audio and video, send the request for video too.
598  // However, don't wait for the response so that we won't have lots of
599  // noticeable pauses in the audio. Video will sync with audio by itself.
600  if (HasVideo() && !HasVideoData()) {
601    RequestVideoData();
602    if (!HasAudio())
603      return;
604  }
605  if (HasAudio() && !HasAudioData()) {
606    RequestAudioData();
607    return;
608  }
609  start_time_ticks_ = base::TimeTicks::Now();
610  start_presentation_timestamp_ = GetCurrentTime();
611  if (!clock_.IsPlaying())
612    clock_.Play();
613  if (HasAudioData() && !audio_decoder_job_->is_decoding())
614    DecodeMoreAudio();
615  if (HasVideoData() && !video_decoder_job_->is_decoding())
616    DecodeMoreVideo();
617  sync_decoder_jobs_ = false;
618}
619
620void MediaSourcePlayer::RequestAudioData() {
621  DVLOG(2) << "RequestAudioData()";
622  DCHECK(HasAudio());
623
624  if (waiting_for_audio_data_)
625    return;
626
627  manager()->OnReadFromDemuxer(player_id(), DemuxerStream::AUDIO);
628  received_audio_ = MediaPlayerHostMsg_ReadFromDemuxerAck_Params();
629  audio_access_unit_index_ = 0;
630  waiting_for_audio_data_ = true;
631}
632
633void MediaSourcePlayer::RequestVideoData() {
634  DVLOG(2) << "RequestVideoData()";
635  DCHECK(HasVideo());
636  if (waiting_for_video_data_)
637    return;
638
639  manager()->OnReadFromDemuxer(player_id(), DemuxerStream::VIDEO);
640  received_video_ = MediaPlayerHostMsg_ReadFromDemuxerAck_Params();
641  video_access_unit_index_ = 0;
642  waiting_for_video_data_ = true;
643}
644
645bool MediaSourcePlayer::HasAudioData() const {
646  return audio_access_unit_index_ < received_audio_.access_units.size();
647}
648
649bool MediaSourcePlayer::HasVideoData() const {
650  return video_access_unit_index_ < received_video_.access_units.size();
651}
652
653void MediaSourcePlayer::SetVolumeInternal() {
654  if (audio_decoder_job_ && volume_ >= 0)
655    audio_decoder_job_.get()->SetVolume(volume_);
656}
657
658}  // namespace media
659