NuPlayer.cpp revision 6ffb1fd67eb8f00f130a6db914ba42a8432aec70
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "NuPlayer" 19#include <utils/Log.h> 20 21#include "NuPlayer.h" 22 23#include "HTTPLiveSource.h" 24#include "NuPlayerDecoder.h" 25#include "NuPlayerDecoderPassThrough.h" 26#include "NuPlayerDriver.h" 27#include "NuPlayerRenderer.h" 28#include "NuPlayerSource.h" 29#include "RTSPSource.h" 30#include "StreamingSource.h" 31#include "GenericSource.h" 32#include "TextDescriptions.h" 33 34#include "ATSParser.h" 35 36#include <media/stagefright/foundation/hexdump.h> 37#include <media/stagefright/foundation/ABuffer.h> 38#include <media/stagefright/foundation/ADebug.h> 39#include <media/stagefright/foundation/AMessage.h> 40#include <media/stagefright/MediaBuffer.h> 41#include <media/stagefright/MediaDefs.h> 42#include <media/stagefright/MediaErrors.h> 43#include <media/stagefright/MetaData.h> 44#include <gui/IGraphicBufferProducer.h> 45 46#include "avc_utils.h" 47 48#include "ESDS.h" 49#include <media/stagefright/Utils.h> 50 51namespace android { 52 53// TODO optimize buffer size for power consumption 54// The offload read buffer size is 32 KB but 24 KB uses less power. 55const size_t NuPlayer::kAggregateBufferSizeBytes = 24 * 1024; 56 57struct NuPlayer::Action : public RefBase { 58 Action() {} 59 60 virtual void execute(NuPlayer *player) = 0; 61 62private: 63 DISALLOW_EVIL_CONSTRUCTORS(Action); 64}; 65 66struct NuPlayer::SeekAction : public Action { 67 SeekAction(int64_t seekTimeUs, bool needNotify) 68 : mSeekTimeUs(seekTimeUs), 69 mNeedNotify(needNotify) { 70 } 71 72 virtual void execute(NuPlayer *player) { 73 player->performSeek(mSeekTimeUs, mNeedNotify); 74 } 75 76private: 77 int64_t mSeekTimeUs; 78 bool mNeedNotify; 79 80 DISALLOW_EVIL_CONSTRUCTORS(SeekAction); 81}; 82 83struct NuPlayer::SetSurfaceAction : public Action { 84 SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper) 85 : mWrapper(wrapper) { 86 } 87 88 virtual void execute(NuPlayer *player) { 89 player->performSetSurface(mWrapper); 90 } 91 92private: 93 sp<NativeWindowWrapper> mWrapper; 94 95 DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction); 96}; 97 98struct NuPlayer::FlushDecoderAction : public Action { 99 FlushDecoderAction(FlushCommand audio, FlushCommand video) 100 : mAudio(audio), 101 mVideo(video) { 102 } 103 104 virtual void execute(NuPlayer *player) { 105 player->performDecoderFlush(mAudio, mVideo); 106 } 107 108private: 109 FlushCommand mAudio; 110 FlushCommand mVideo; 111 112 DISALLOW_EVIL_CONSTRUCTORS(FlushDecoderAction); 113}; 114 115struct NuPlayer::PostMessageAction : public Action { 116 PostMessageAction(const sp<AMessage> &msg) 117 : mMessage(msg) { 118 } 119 120 virtual void execute(NuPlayer *) { 121 mMessage->post(); 122 } 123 124private: 125 sp<AMessage> mMessage; 126 127 DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction); 128}; 129 130// Use this if there's no state necessary to save in order to execute 131// the action. 132struct NuPlayer::SimpleAction : public Action { 133 typedef void (NuPlayer::*ActionFunc)(); 134 135 SimpleAction(ActionFunc func) 136 : mFunc(func) { 137 } 138 139 virtual void execute(NuPlayer *player) { 140 (player->*mFunc)(); 141 } 142 143private: 144 ActionFunc mFunc; 145 146 DISALLOW_EVIL_CONSTRUCTORS(SimpleAction); 147}; 148 149//////////////////////////////////////////////////////////////////////////////// 150 151NuPlayer::NuPlayer() 152 : mUIDValid(false), 153 mSourceFlags(0), 154 mVideoIsAVC(false), 155 mOffloadAudio(false), 156 mAudioDecoderGeneration(0), 157 mVideoDecoderGeneration(0), 158 mRendererGeneration(0), 159 mAudioEOS(false), 160 mVideoEOS(false), 161 mScanSourcesPending(false), 162 mScanSourcesGeneration(0), 163 mPollDurationGeneration(0), 164 mTimedTextGeneration(0), 165 mTimeDiscontinuityPending(false), 166 mFlushingAudio(NONE), 167 mFlushingVideo(NONE), 168 mNumFramesTotal(0ll), 169 mNumFramesDropped(0ll), 170 mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW), 171 mStarted(false) { 172 clearFlushComplete(); 173} 174 175NuPlayer::~NuPlayer() { 176} 177 178void NuPlayer::setUID(uid_t uid) { 179 mUIDValid = true; 180 mUID = uid; 181} 182 183void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) { 184 mDriver = driver; 185} 186 187void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) { 188 sp<AMessage> msg = new AMessage(kWhatSetDataSource, id()); 189 190 sp<AMessage> notify = new AMessage(kWhatSourceNotify, id()); 191 192 msg->setObject("source", new StreamingSource(notify, source)); 193 msg->post(); 194} 195 196static bool IsHTTPLiveURL(const char *url) { 197 if (!strncasecmp("http://", url, 7) 198 || !strncasecmp("https://", url, 8) 199 || !strncasecmp("file://", url, 7)) { 200 size_t len = strlen(url); 201 if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) { 202 return true; 203 } 204 205 if (strstr(url,"m3u8")) { 206 return true; 207 } 208 } 209 210 return false; 211} 212 213void NuPlayer::setDataSourceAsync( 214 const sp<IMediaHTTPService> &httpService, 215 const char *url, 216 const KeyedVector<String8, String8> *headers) { 217 218 sp<AMessage> msg = new AMessage(kWhatSetDataSource, id()); 219 size_t len = strlen(url); 220 221 sp<AMessage> notify = new AMessage(kWhatSourceNotify, id()); 222 223 sp<Source> source; 224 if (IsHTTPLiveURL(url)) { 225 source = new HTTPLiveSource(notify, httpService, url, headers); 226 } else if (!strncasecmp(url, "rtsp://", 7)) { 227 source = new RTSPSource( 228 notify, httpService, url, headers, mUIDValid, mUID); 229 } else if ((!strncasecmp(url, "http://", 7) 230 || !strncasecmp(url, "https://", 8)) 231 && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4])) 232 || strstr(url, ".sdp?"))) { 233 source = new RTSPSource( 234 notify, httpService, url, headers, mUIDValid, mUID, true); 235 } else { 236 sp<GenericSource> genericSource = 237 new GenericSource(notify, mUIDValid, mUID); 238 // Don't set FLAG_SECURE on mSourceFlags here for widevine. 239 // The correct flags will be updated in Source::kWhatFlagsChanged 240 // handler when GenericSource is prepared. 241 242 status_t err = genericSource->setDataSource(httpService, url, headers); 243 244 if (err == OK) { 245 source = genericSource; 246 } else { 247 ALOGE("Failed to set data source!"); 248 } 249 } 250 msg->setObject("source", source); 251 msg->post(); 252} 253 254void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) { 255 sp<AMessage> msg = new AMessage(kWhatSetDataSource, id()); 256 257 sp<AMessage> notify = new AMessage(kWhatSourceNotify, id()); 258 259 sp<GenericSource> source = 260 new GenericSource(notify, mUIDValid, mUID); 261 262 status_t err = source->setDataSource(fd, offset, length); 263 264 if (err != OK) { 265 ALOGE("Failed to set data source!"); 266 source = NULL; 267 } 268 269 msg->setObject("source", source); 270 msg->post(); 271} 272 273void NuPlayer::prepareAsync() { 274 (new AMessage(kWhatPrepare, id()))->post(); 275} 276 277void NuPlayer::setVideoSurfaceTextureAsync( 278 const sp<IGraphicBufferProducer> &bufferProducer) { 279 sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, id()); 280 281 if (bufferProducer == NULL) { 282 msg->setObject("native-window", NULL); 283 } else { 284 msg->setObject( 285 "native-window", 286 new NativeWindowWrapper( 287 new Surface(bufferProducer, true /* controlledByApp */))); 288 } 289 290 msg->post(); 291} 292 293void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) { 294 sp<AMessage> msg = new AMessage(kWhatSetAudioSink, id()); 295 msg->setObject("sink", sink); 296 msg->post(); 297} 298 299void NuPlayer::start() { 300 (new AMessage(kWhatStart, id()))->post(); 301} 302 303void NuPlayer::pause() { 304 (new AMessage(kWhatPause, id()))->post(); 305} 306 307void NuPlayer::resetAsync() { 308 if (mSource != NULL) { 309 // During a reset, the data source might be unresponsive already, we need to 310 // disconnect explicitly so that reads exit promptly. 311 // We can't queue the disconnect request to the looper, as it might be 312 // queued behind a stuck read and never gets processed. 313 // Doing a disconnect outside the looper to allows the pending reads to exit 314 // (either successfully or with error). 315 mSource->disconnect(); 316 } 317 318 (new AMessage(kWhatReset, id()))->post(); 319} 320 321void NuPlayer::seekToAsync(int64_t seekTimeUs, bool needNotify) { 322 sp<AMessage> msg = new AMessage(kWhatSeek, id()); 323 msg->setInt64("seekTimeUs", seekTimeUs); 324 msg->setInt32("needNotify", needNotify); 325 msg->post(); 326} 327 328 329void NuPlayer::writeTrackInfo( 330 Parcel* reply, const sp<AMessage> format) const { 331 int32_t trackType; 332 CHECK(format->findInt32("type", &trackType)); 333 334 AString lang; 335 CHECK(format->findString("language", &lang)); 336 337 reply->writeInt32(2); // write something non-zero 338 reply->writeInt32(trackType); 339 reply->writeString16(String16(lang.c_str())); 340 341 if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) { 342 AString mime; 343 CHECK(format->findString("mime", &mime)); 344 345 int32_t isAuto, isDefault, isForced; 346 CHECK(format->findInt32("auto", &isAuto)); 347 CHECK(format->findInt32("default", &isDefault)); 348 CHECK(format->findInt32("forced", &isForced)); 349 350 reply->writeString16(String16(mime.c_str())); 351 reply->writeInt32(isAuto); 352 reply->writeInt32(isDefault); 353 reply->writeInt32(isForced); 354 } 355} 356 357void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { 358 switch (msg->what()) { 359 case kWhatSetDataSource: 360 { 361 ALOGV("kWhatSetDataSource"); 362 363 CHECK(mSource == NULL); 364 365 status_t err = OK; 366 sp<RefBase> obj; 367 CHECK(msg->findObject("source", &obj)); 368 if (obj != NULL) { 369 mSource = static_cast<Source *>(obj.get()); 370 } else { 371 err = UNKNOWN_ERROR; 372 } 373 374 CHECK(mDriver != NULL); 375 sp<NuPlayerDriver> driver = mDriver.promote(); 376 if (driver != NULL) { 377 driver->notifySetDataSourceCompleted(err); 378 } 379 break; 380 } 381 382 case kWhatPrepare: 383 { 384 mSource->prepareAsync(); 385 break; 386 } 387 388 case kWhatGetTrackInfo: 389 { 390 uint32_t replyID; 391 CHECK(msg->senderAwaitsResponse(&replyID)); 392 393 Parcel* reply; 394 CHECK(msg->findPointer("reply", (void**)&reply)); 395 396 size_t inbandTracks = 0; 397 if (mSource != NULL) { 398 inbandTracks = mSource->getTrackCount(); 399 } 400 401 size_t ccTracks = 0; 402 if (mCCDecoder != NULL) { 403 ccTracks = mCCDecoder->getTrackCount(); 404 } 405 406 // total track count 407 reply->writeInt32(inbandTracks + ccTracks); 408 409 // write inband tracks 410 for (size_t i = 0; i < inbandTracks; ++i) { 411 writeTrackInfo(reply, mSource->getTrackInfo(i)); 412 } 413 414 // write CC track 415 for (size_t i = 0; i < ccTracks; ++i) { 416 writeTrackInfo(reply, mCCDecoder->getTrackInfo(i)); 417 } 418 419 sp<AMessage> response = new AMessage; 420 response->postReply(replyID); 421 break; 422 } 423 424 case kWhatGetSelectedTrack: 425 { 426 status_t err = INVALID_OPERATION; 427 if (mSource != NULL) { 428 err = OK; 429 430 int32_t type32; 431 CHECK(msg->findInt32("type", (int32_t*)&type32)); 432 media_track_type type = (media_track_type)type32; 433 ssize_t selectedTrack = mSource->getSelectedTrack(type); 434 435 Parcel* reply; 436 CHECK(msg->findPointer("reply", (void**)&reply)); 437 reply->writeInt32(selectedTrack); 438 } 439 440 sp<AMessage> response = new AMessage; 441 response->setInt32("err", err); 442 443 uint32_t replyID; 444 CHECK(msg->senderAwaitsResponse(&replyID)); 445 response->postReply(replyID); 446 break; 447 } 448 449 case kWhatSelectTrack: 450 { 451 uint32_t replyID; 452 CHECK(msg->senderAwaitsResponse(&replyID)); 453 454 size_t trackIndex; 455 int32_t select; 456 int64_t timeUs; 457 CHECK(msg->findSize("trackIndex", &trackIndex)); 458 CHECK(msg->findInt32("select", &select)); 459 CHECK(msg->findInt64("timeUs", &timeUs)); 460 461 status_t err = INVALID_OPERATION; 462 463 size_t inbandTracks = 0; 464 if (mSource != NULL) { 465 inbandTracks = mSource->getTrackCount(); 466 } 467 size_t ccTracks = 0; 468 if (mCCDecoder != NULL) { 469 ccTracks = mCCDecoder->getTrackCount(); 470 } 471 472 if (trackIndex < inbandTracks) { 473 err = mSource->selectTrack(trackIndex, select, timeUs); 474 475 if (!select && err == OK) { 476 int32_t type; 477 sp<AMessage> info = mSource->getTrackInfo(trackIndex); 478 if (info != NULL 479 && info->findInt32("type", &type) 480 && type == MEDIA_TRACK_TYPE_TIMEDTEXT) { 481 ++mTimedTextGeneration; 482 } 483 } 484 } else { 485 trackIndex -= inbandTracks; 486 487 if (trackIndex < ccTracks) { 488 err = mCCDecoder->selectTrack(trackIndex, select); 489 } 490 } 491 492 sp<AMessage> response = new AMessage; 493 response->setInt32("err", err); 494 495 response->postReply(replyID); 496 break; 497 } 498 499 case kWhatPollDuration: 500 { 501 int32_t generation; 502 CHECK(msg->findInt32("generation", &generation)); 503 504 if (generation != mPollDurationGeneration) { 505 // stale 506 break; 507 } 508 509 int64_t durationUs; 510 if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) { 511 sp<NuPlayerDriver> driver = mDriver.promote(); 512 if (driver != NULL) { 513 driver->notifyDuration(durationUs); 514 } 515 } 516 517 msg->post(1000000ll); // poll again in a second. 518 break; 519 } 520 521 case kWhatSetVideoNativeWindow: 522 { 523 ALOGV("kWhatSetVideoNativeWindow"); 524 525 sp<RefBase> obj; 526 CHECK(msg->findObject("native-window", &obj)); 527 528 if (mSource->getFormat(false /* audio */) == NULL) { 529 performSetSurface(static_cast<NativeWindowWrapper *>(obj.get())); 530 break; 531 } 532 533 mDeferredActions.push_back( 534 new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */, 535 FLUSH_CMD_SHUTDOWN /* video */)); 536 537 mDeferredActions.push_back( 538 new SetSurfaceAction( 539 static_cast<NativeWindowWrapper *>(obj.get()))); 540 541 if (obj != NULL) { 542 if (mStarted) { 543 // Issue a seek to refresh the video screen only if started otherwise 544 // the extractor may not yet be started and will assert. 545 // If the video decoder is not set (perhaps audio only in this case) 546 // do not perform a seek as it is not needed. 547 int64_t currentPositionUs = 0; 548 if (getCurrentPosition(¤tPositionUs) == OK) { 549 mDeferredActions.push_back( 550 new SeekAction(currentPositionUs, false /* needNotify */)); 551 } 552 } 553 554 // If there is a new surface texture, instantiate decoders 555 // again if possible. 556 mDeferredActions.push_back( 557 new SimpleAction(&NuPlayer::performScanSources)); 558 } 559 560 processDeferredActions(); 561 break; 562 } 563 564 case kWhatSetAudioSink: 565 { 566 ALOGV("kWhatSetAudioSink"); 567 568 sp<RefBase> obj; 569 CHECK(msg->findObject("sink", &obj)); 570 571 mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get()); 572 break; 573 } 574 575 case kWhatStart: 576 { 577 ALOGV("kWhatStart"); 578 if (mStarted) { 579 onResume(); 580 } else { 581 onStart(); 582 } 583 break; 584 } 585 586 case kWhatScanSources: 587 { 588 int32_t generation; 589 CHECK(msg->findInt32("generation", &generation)); 590 if (generation != mScanSourcesGeneration) { 591 // Drop obsolete msg. 592 break; 593 } 594 595 mScanSourcesPending = false; 596 597 ALOGV("scanning sources haveAudio=%d, haveVideo=%d", 598 mAudioDecoder != NULL, mVideoDecoder != NULL); 599 600 bool mHadAnySourcesBefore = 601 (mAudioDecoder != NULL) || (mVideoDecoder != NULL); 602 603 // initialize video before audio because successful initialization of 604 // video may change deep buffer mode of audio. 605 if (mNativeWindow != NULL) { 606 instantiateDecoder(false, &mVideoDecoder); 607 } 608 609 if (mAudioSink != NULL) { 610 if (mOffloadAudio) { 611 // open audio sink early under offload mode. 612 sp<AMessage> format = mSource->getFormat(true /*audio*/); 613 openAudioSink(format, true /*offloadOnly*/); 614 } 615 instantiateDecoder(true, &mAudioDecoder); 616 } 617 618 if (!mHadAnySourcesBefore 619 && (mAudioDecoder != NULL || mVideoDecoder != NULL)) { 620 // This is the first time we've found anything playable. 621 622 if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) { 623 schedulePollDuration(); 624 } 625 } 626 627 status_t err; 628 if ((err = mSource->feedMoreTSData()) != OK) { 629 if (mAudioDecoder == NULL && mVideoDecoder == NULL) { 630 // We're not currently decoding anything (no audio or 631 // video tracks found) and we just ran out of input data. 632 633 if (err == ERROR_END_OF_STREAM) { 634 notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0); 635 } else { 636 notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err); 637 } 638 } 639 break; 640 } 641 642 if ((mAudioDecoder == NULL && mAudioSink != NULL) 643 || (mVideoDecoder == NULL && mNativeWindow != NULL)) { 644 msg->post(100000ll); 645 mScanSourcesPending = true; 646 } 647 break; 648 } 649 650 case kWhatVideoNotify: 651 case kWhatAudioNotify: 652 { 653 bool audio = msg->what() == kWhatAudioNotify; 654 655 int32_t currentDecoderGeneration = 656 (audio? mAudioDecoderGeneration : mVideoDecoderGeneration); 657 int32_t requesterGeneration = currentDecoderGeneration - 1; 658 CHECK(msg->findInt32("generation", &requesterGeneration)); 659 660 if (requesterGeneration != currentDecoderGeneration) { 661 ALOGV("got message from old %s decoder, generation(%d:%d)", 662 audio ? "audio" : "video", requesterGeneration, 663 currentDecoderGeneration); 664 sp<AMessage> reply; 665 if (!(msg->findMessage("reply", &reply))) { 666 return; 667 } 668 669 reply->setInt32("err", INFO_DISCONTINUITY); 670 reply->post(); 671 return; 672 } 673 674 int32_t what; 675 CHECK(msg->findInt32("what", &what)); 676 677 if (what == Decoder::kWhatFillThisBuffer) { 678 status_t err = feedDecoderInputData( 679 audio, msg); 680 681 if (err == -EWOULDBLOCK) { 682 if (mSource->feedMoreTSData() == OK) { 683 msg->post(10 * 1000ll); 684 } 685 } 686 } else if (what == Decoder::kWhatEOS) { 687 int32_t err; 688 CHECK(msg->findInt32("err", &err)); 689 690 if (err == ERROR_END_OF_STREAM) { 691 ALOGV("got %s decoder EOS", audio ? "audio" : "video"); 692 } else { 693 ALOGV("got %s decoder EOS w/ error %d", 694 audio ? "audio" : "video", 695 err); 696 } 697 698 mRenderer->queueEOS(audio, err); 699 } else if (what == Decoder::kWhatFlushCompleted) { 700 ALOGV("decoder %s flush completed", audio ? "audio" : "video"); 701 702 handleFlushComplete(audio, true /* isDecoder */); 703 finishFlushIfPossible(); 704 } else if (what == Decoder::kWhatVideoSizeChanged) { 705 sp<AMessage> format; 706 CHECK(msg->findMessage("format", &format)); 707 708 sp<AMessage> inputFormat = 709 mSource->getFormat(false /* audio */); 710 711 updateVideoSize(inputFormat, format); 712 } else if (what == Decoder::kWhatShutdownCompleted) { 713 ALOGV("%s shutdown completed", audio ? "audio" : "video"); 714 if (audio) { 715 mAudioDecoder.clear(); 716 ++mAudioDecoderGeneration; 717 718 CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER); 719 mFlushingAudio = SHUT_DOWN; 720 } else { 721 mVideoDecoder.clear(); 722 ++mVideoDecoderGeneration; 723 724 CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER); 725 mFlushingVideo = SHUT_DOWN; 726 } 727 728 finishFlushIfPossible(); 729 } else if (what == Decoder::kWhatError) { 730 status_t err; 731 if (!msg->findInt32("err", &err) || err == OK) { 732 err = UNKNOWN_ERROR; 733 } 734 735 // Decoder errors can be due to Source (e.g. from streaming), 736 // or from decoding corrupted bitstreams, or from other decoder 737 // MediaCodec operations (e.g. from an ongoing reset or seek). 738 // 739 // We try to gracefully shut down the affected decoder if possible, 740 // rather than trying to force the shutdown with something 741 // similar to performReset(). This method can lead to a hang 742 // if MediaCodec functions block after an error, but they should 743 // typically return INVALID_OPERATION instead of blocking. 744 745 FlushStatus *flushing = audio ? &mFlushingAudio : &mFlushingVideo; 746 ALOGE("received error(%#x) from %s decoder, flushing(%d), now shutting down", 747 err, audio ? "audio" : "video", *flushing); 748 749 switch (*flushing) { 750 case NONE: 751 mDeferredActions.push_back( 752 new FlushDecoderAction( 753 audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE, 754 audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN)); 755 processDeferredActions(); 756 break; 757 case FLUSHING_DECODER: 758 *flushing = FLUSHING_DECODER_SHUTDOWN; // initiate shutdown after flush. 759 break; // Wait for flush to complete. 760 case FLUSHING_DECODER_SHUTDOWN: 761 break; // Wait for flush to complete. 762 case SHUTTING_DOWN_DECODER: 763 break; // Wait for shutdown to complete. 764 case FLUSHED: 765 // Widevine source reads must stop before releasing the video decoder. 766 if (!audio && mSource != NULL && mSourceFlags & Source::FLAG_SECURE) { 767 mSource->stop(); 768 } 769 getDecoder(audio)->initiateShutdown(); // In the middle of a seek. 770 *flushing = SHUTTING_DOWN_DECODER; // Shut down. 771 break; 772 case SHUT_DOWN: 773 finishFlushIfPossible(); // Should not occur. 774 break; // Finish anyways. 775 } 776 notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err); 777 } else if (what == Decoder::kWhatRenderBufferTime) { 778 renderBuffer(audio, msg); 779 } else { 780 ALOGV("Unhandled decoder notification %d '%c%c%c%c'.", 781 what, 782 what >> 24, 783 (what >> 16) & 0xff, 784 (what >> 8) & 0xff, 785 what & 0xff); 786 } 787 788 break; 789 } 790 791 case kWhatRendererNotify: 792 { 793 int32_t requesterGeneration = mRendererGeneration - 1; 794 CHECK(msg->findInt32("generation", &requesterGeneration)); 795 if (requesterGeneration != mRendererGeneration) { 796 ALOGV("got message from old renderer, generation(%d:%d)", 797 requesterGeneration, mRendererGeneration); 798 return; 799 } 800 801 int32_t what; 802 CHECK(msg->findInt32("what", &what)); 803 804 if (what == Renderer::kWhatEOS) { 805 int32_t audio; 806 CHECK(msg->findInt32("audio", &audio)); 807 808 int32_t finalResult; 809 CHECK(msg->findInt32("finalResult", &finalResult)); 810 811 if (audio) { 812 mAudioEOS = true; 813 } else { 814 mVideoEOS = true; 815 } 816 817 if (finalResult == ERROR_END_OF_STREAM) { 818 ALOGV("reached %s EOS", audio ? "audio" : "video"); 819 } else { 820 ALOGE("%s track encountered an error (%d)", 821 audio ? "audio" : "video", finalResult); 822 823 notifyListener( 824 MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult); 825 } 826 827 if ((mAudioEOS || mAudioDecoder == NULL) 828 && (mVideoEOS || mVideoDecoder == NULL)) { 829 notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0); 830 } 831 } else if (what == Renderer::kWhatFlushComplete) { 832 int32_t audio; 833 CHECK(msg->findInt32("audio", &audio)); 834 835 ALOGV("renderer %s flush completed.", audio ? "audio" : "video"); 836 handleFlushComplete(audio, false /* isDecoder */); 837 finishFlushIfPossible(); 838 } else if (what == Renderer::kWhatVideoRenderingStart) { 839 notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0); 840 } else if (what == Renderer::kWhatMediaRenderingStart) { 841 ALOGV("media rendering started"); 842 notifyListener(MEDIA_STARTED, 0, 0); 843 } else if (what == Renderer::kWhatAudioOffloadTearDown) { 844 ALOGV("Tear down audio offload, fall back to s/w path"); 845 int64_t positionUs; 846 CHECK(msg->findInt64("positionUs", &positionUs)); 847 int32_t reason; 848 CHECK(msg->findInt32("reason", &reason)); 849 closeAudioSink(); 850 mAudioDecoder.clear(); 851 ++mAudioDecoderGeneration; 852 mRenderer->flush(true /* audio */); 853 if (mVideoDecoder != NULL) { 854 mRenderer->flush(false /* audio */); 855 } 856 mRenderer->signalDisableOffloadAudio(); 857 mOffloadAudio = false; 858 859 performSeek(positionUs, false /* needNotify */); 860 if (reason == Renderer::kDueToError) { 861 instantiateDecoder(true /* audio */, &mAudioDecoder); 862 } 863 } 864 break; 865 } 866 867 case kWhatMoreDataQueued: 868 { 869 break; 870 } 871 872 case kWhatReset: 873 { 874 ALOGV("kWhatReset"); 875 876 mDeferredActions.push_back( 877 new FlushDecoderAction( 878 FLUSH_CMD_SHUTDOWN /* audio */, 879 FLUSH_CMD_SHUTDOWN /* video */)); 880 881 mDeferredActions.push_back( 882 new SimpleAction(&NuPlayer::performReset)); 883 884 processDeferredActions(); 885 break; 886 } 887 888 case kWhatSeek: 889 { 890 int64_t seekTimeUs; 891 int32_t needNotify; 892 CHECK(msg->findInt64("seekTimeUs", &seekTimeUs)); 893 CHECK(msg->findInt32("needNotify", &needNotify)); 894 895 ALOGV("kWhatSeek seekTimeUs=%lld us, needNotify=%d", 896 seekTimeUs, needNotify); 897 898 mDeferredActions.push_back( 899 new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */, 900 FLUSH_CMD_FLUSH /* video */)); 901 902 mDeferredActions.push_back( 903 new SeekAction(seekTimeUs, needNotify)); 904 905 processDeferredActions(); 906 break; 907 } 908 909 case kWhatPause: 910 { 911 if (mSource != NULL) { 912 mSource->pause(); 913 } else { 914 ALOGW("pause called when source is gone or not set"); 915 } 916 if (mRenderer != NULL) { 917 mRenderer->pause(); 918 } else { 919 ALOGW("pause called when renderer is gone or not set"); 920 } 921 break; 922 } 923 924 case kWhatSourceNotify: 925 { 926 onSourceNotify(msg); 927 break; 928 } 929 930 case kWhatClosedCaptionNotify: 931 { 932 onClosedCaptionNotify(msg); 933 break; 934 } 935 936 default: 937 TRESPASS(); 938 break; 939 } 940} 941 942void NuPlayer::onResume() { 943 if (mSource != NULL) { 944 mSource->resume(); 945 } else { 946 ALOGW("resume called when source is gone or not set"); 947 } 948 // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if 949 // needed. 950 if (audioDecoderStillNeeded() && mAudioDecoder == NULL) { 951 instantiateDecoder(true /* audio */, &mAudioDecoder); 952 } 953 if (mRenderer != NULL) { 954 mRenderer->resume(); 955 } else { 956 ALOGW("resume called when renderer is gone or not set"); 957 } 958} 959 960void NuPlayer::onStart() { 961 mVideoIsAVC = false; 962 mOffloadAudio = false; 963 mAudioEOS = false; 964 mVideoEOS = false; 965 mNumFramesTotal = 0; 966 mNumFramesDropped = 0; 967 mStarted = true; 968 969 /* instantiate decoders now for secure playback */ 970 if (mSourceFlags & Source::FLAG_SECURE) { 971 if (mNativeWindow != NULL) { 972 instantiateDecoder(false, &mVideoDecoder); 973 } 974 975 if (mAudioSink != NULL) { 976 instantiateDecoder(true, &mAudioDecoder); 977 } 978 } 979 980 mSource->start(); 981 982 uint32_t flags = 0; 983 984 if (mSource->isRealTime()) { 985 flags |= Renderer::FLAG_REAL_TIME; 986 } 987 988 sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */); 989 audio_stream_type_t streamType = AUDIO_STREAM_MUSIC; 990 if (mAudioSink != NULL) { 991 streamType = mAudioSink->getAudioStreamType(); 992 } 993 994 sp<AMessage> videoFormat = mSource->getFormat(false /* audio */); 995 996 mOffloadAudio = 997 canOffloadStream(audioMeta, (videoFormat != NULL), 998 true /* is_streaming */, streamType); 999 if (mOffloadAudio) { 1000 flags |= Renderer::FLAG_OFFLOAD_AUDIO; 1001 } 1002 1003 sp<AMessage> notify = new AMessage(kWhatRendererNotify, id()); 1004 ++mRendererGeneration; 1005 notify->setInt32("generation", mRendererGeneration); 1006 mRenderer = new Renderer(mAudioSink, notify, flags); 1007 1008 mRendererLooper = new ALooper; 1009 mRendererLooper->setName("NuPlayerRenderer"); 1010 mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 1011 mRendererLooper->registerHandler(mRenderer); 1012 1013 sp<MetaData> meta = getFileMeta(); 1014 int32_t rate; 1015 if (meta != NULL 1016 && meta->findInt32(kKeyFrameRate, &rate) && rate > 0) { 1017 mRenderer->setVideoFrameRate(rate); 1018 } 1019 1020 if (mVideoDecoder != NULL) { 1021 mVideoDecoder->setRenderer(mRenderer); 1022 } 1023 if (mAudioDecoder != NULL) { 1024 mAudioDecoder->setRenderer(mRenderer); 1025 } 1026 1027 postScanSources(); 1028} 1029 1030bool NuPlayer::audioDecoderStillNeeded() { 1031 // Audio decoder is no longer needed if it's in shut/shutting down status. 1032 return ((mFlushingAudio != SHUT_DOWN) && (mFlushingAudio != SHUTTING_DOWN_DECODER)); 1033} 1034 1035void NuPlayer::handleFlushComplete(bool audio, bool isDecoder) { 1036 // We wait for both the decoder flush and the renderer flush to complete 1037 // before entering either the FLUSHED or the SHUTTING_DOWN_DECODER state. 1038 1039 mFlushComplete[audio][isDecoder] = true; 1040 if (!mFlushComplete[audio][!isDecoder]) { 1041 return; 1042 } 1043 1044 FlushStatus *state = audio ? &mFlushingAudio : &mFlushingVideo; 1045 switch (*state) { 1046 case FLUSHING_DECODER: 1047 { 1048 *state = FLUSHED; 1049 break; 1050 } 1051 1052 case FLUSHING_DECODER_SHUTDOWN: 1053 { 1054 *state = SHUTTING_DOWN_DECODER; 1055 1056 ALOGV("initiating %s decoder shutdown", audio ? "audio" : "video"); 1057 if (!audio) { 1058 // Widevine source reads must stop before releasing the video decoder. 1059 if (mSource != NULL && mSourceFlags & Source::FLAG_SECURE) { 1060 mSource->stop(); 1061 } 1062 } 1063 getDecoder(audio)->initiateShutdown(); 1064 break; 1065 } 1066 1067 default: 1068 // decoder flush completes only occur in a flushing state. 1069 LOG_ALWAYS_FATAL_IF(isDecoder, "decoder flush in invalid state %d", *state); 1070 break; 1071 } 1072} 1073 1074void NuPlayer::finishFlushIfPossible() { 1075 if (mFlushingAudio != NONE && mFlushingAudio != FLUSHED 1076 && mFlushingAudio != SHUT_DOWN) { 1077 return; 1078 } 1079 1080 if (mFlushingVideo != NONE && mFlushingVideo != FLUSHED 1081 && mFlushingVideo != SHUT_DOWN) { 1082 return; 1083 } 1084 1085 ALOGV("both audio and video are flushed now."); 1086 1087 mPendingAudioAccessUnit.clear(); 1088 mAggregateBuffer.clear(); 1089 1090 if (mTimeDiscontinuityPending) { 1091 mRenderer->signalTimeDiscontinuity(); 1092 mTimeDiscontinuityPending = false; 1093 } 1094 1095 if (mAudioDecoder != NULL && mFlushingAudio == FLUSHED) { 1096 mAudioDecoder->signalResume(); 1097 } 1098 1099 if (mVideoDecoder != NULL && mFlushingVideo == FLUSHED) { 1100 mVideoDecoder->signalResume(); 1101 } 1102 1103 mFlushingAudio = NONE; 1104 mFlushingVideo = NONE; 1105 1106 clearFlushComplete(); 1107 1108 processDeferredActions(); 1109} 1110 1111void NuPlayer::postScanSources() { 1112 if (mScanSourcesPending) { 1113 return; 1114 } 1115 1116 sp<AMessage> msg = new AMessage(kWhatScanSources, id()); 1117 msg->setInt32("generation", mScanSourcesGeneration); 1118 msg->post(); 1119 1120 mScanSourcesPending = true; 1121} 1122 1123void NuPlayer::openAudioSink(const sp<AMessage> &format, bool offloadOnly) { 1124 uint32_t flags; 1125 int64_t durationUs; 1126 bool hasVideo = (mVideoDecoder != NULL); 1127 // FIXME: we should handle the case where the video decoder 1128 // is created after we receive the format change indication. 1129 // Current code will just make that we select deep buffer 1130 // with video which should not be a problem as it should 1131 // not prevent from keeping A/V sync. 1132 if (!hasVideo && 1133 mSource->getDuration(&durationUs) == OK && 1134 durationUs 1135 > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) { 1136 flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER; 1137 } else { 1138 flags = AUDIO_OUTPUT_FLAG_NONE; 1139 } 1140 1141 mOffloadAudio = mRenderer->openAudioSink( 1142 format, offloadOnly, hasVideo, flags); 1143 1144 if (mOffloadAudio) { 1145 sp<MetaData> audioMeta = 1146 mSource->getFormatMeta(true /* audio */); 1147 sendMetaDataToHal(mAudioSink, audioMeta); 1148 } 1149} 1150 1151void NuPlayer::closeAudioSink() { 1152 mRenderer->closeAudioSink(); 1153} 1154 1155status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) { 1156 if (*decoder != NULL) { 1157 return OK; 1158 } 1159 1160 sp<AMessage> format = mSource->getFormat(audio); 1161 1162 if (format == NULL) { 1163 return -EWOULDBLOCK; 1164 } 1165 1166 if (!audio) { 1167 AString mime; 1168 CHECK(format->findString("mime", &mime)); 1169 mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str()); 1170 1171 sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id()); 1172 mCCDecoder = new CCDecoder(ccNotify); 1173 1174 if (mSourceFlags & Source::FLAG_SECURE) { 1175 format->setInt32("secure", true); 1176 } 1177 } 1178 1179 if (audio) { 1180 sp<AMessage> notify = new AMessage(kWhatAudioNotify, id()); 1181 ++mAudioDecoderGeneration; 1182 notify->setInt32("generation", mAudioDecoderGeneration); 1183 1184 if (mOffloadAudio) { 1185 *decoder = new DecoderPassThrough(notify, mSource, mRenderer); 1186 } else { 1187 *decoder = new Decoder(notify, mSource, mRenderer); 1188 } 1189 } else { 1190 sp<AMessage> notify = new AMessage(kWhatVideoNotify, id()); 1191 ++mVideoDecoderGeneration; 1192 notify->setInt32("generation", mVideoDecoderGeneration); 1193 1194 *decoder = new Decoder(notify, mSource, mRenderer, mNativeWindow); 1195 } 1196 (*decoder)->init(); 1197 (*decoder)->configure(format); 1198 1199 // allocate buffers to decrypt widevine source buffers 1200 if (!audio && (mSourceFlags & Source::FLAG_SECURE)) { 1201 Vector<sp<ABuffer> > inputBufs; 1202 CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK); 1203 1204 Vector<MediaBuffer *> mediaBufs; 1205 for (size_t i = 0; i < inputBufs.size(); i++) { 1206 const sp<ABuffer> &buffer = inputBufs[i]; 1207 MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size()); 1208 mediaBufs.push(mbuf); 1209 } 1210 1211 status_t err = mSource->setBuffers(audio, mediaBufs); 1212 if (err != OK) { 1213 for (size_t i = 0; i < mediaBufs.size(); ++i) { 1214 mediaBufs[i]->release(); 1215 } 1216 mediaBufs.clear(); 1217 ALOGE("Secure source didn't support secure mediaBufs."); 1218 return err; 1219 } 1220 } 1221 return OK; 1222} 1223 1224status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) { 1225 sp<AMessage> reply; 1226 CHECK(msg->findMessage("reply", &reply)); 1227 1228 if ((audio && mFlushingAudio != NONE) 1229 || (!audio && mFlushingVideo != NONE) 1230 || mSource == NULL) { 1231 reply->setInt32("err", INFO_DISCONTINUITY); 1232 reply->post(); 1233 return OK; 1234 } 1235 1236 sp<ABuffer> accessUnit; 1237 1238 // Aggregate smaller buffers into a larger buffer. 1239 // The goal is to reduce power consumption. 1240 // Note this will not work if the decoder requires one frame per buffer. 1241 bool doBufferAggregation = (audio && mOffloadAudio); 1242 bool needMoreData = false; 1243 1244 bool dropAccessUnit; 1245 do { 1246 status_t err; 1247 // Did we save an accessUnit earlier because of a discontinuity? 1248 if (audio && (mPendingAudioAccessUnit != NULL)) { 1249 accessUnit = mPendingAudioAccessUnit; 1250 mPendingAudioAccessUnit.clear(); 1251 err = mPendingAudioErr; 1252 ALOGV("feedDecoderInputData() use mPendingAudioAccessUnit"); 1253 } else { 1254 err = mSource->dequeueAccessUnit(audio, &accessUnit); 1255 } 1256 1257 if (err == -EWOULDBLOCK) { 1258 return err; 1259 } else if (err != OK) { 1260 if (err == INFO_DISCONTINUITY) { 1261 if (doBufferAggregation && (mAggregateBuffer != NULL)) { 1262 // We already have some data so save this for later. 1263 mPendingAudioErr = err; 1264 mPendingAudioAccessUnit = accessUnit; 1265 accessUnit.clear(); 1266 ALOGD("feedDecoderInputData() save discontinuity for later"); 1267 break; 1268 } 1269 int32_t type; 1270 CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); 1271 1272 bool formatChange = 1273 (audio && 1274 (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT)) 1275 || (!audio && 1276 (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT)); 1277 1278 bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0; 1279 1280 ALOGI("%s discontinuity (formatChange=%d, time=%d)", 1281 audio ? "audio" : "video", formatChange, timeChange); 1282 1283 mTimeDiscontinuityPending = 1284 mTimeDiscontinuityPending || timeChange; 1285 1286 bool seamlessFormatChange = false; 1287 sp<AMessage> newFormat = mSource->getFormat(audio); 1288 if (formatChange) { 1289 seamlessFormatChange = 1290 getDecoder(audio)->supportsSeamlessFormatChange(newFormat); 1291 // treat seamless format change separately 1292 formatChange = !seamlessFormatChange; 1293 } 1294 bool shutdownOrFlush = formatChange || timeChange; 1295 1296 // We want to queue up scan-sources only once per discontinuity. 1297 // We control this by doing it only if neither audio nor video are 1298 // flushing or shutting down. (After handling 1st discontinuity, one 1299 // of the flushing states will not be NONE.) 1300 // No need to scan sources if this discontinuity does not result 1301 // in a flush or shutdown, as the flushing state will stay NONE. 1302 if (mFlushingAudio == NONE && mFlushingVideo == NONE && 1303 shutdownOrFlush) { 1304 // And we'll resume scanning sources once we're done 1305 // flushing. 1306 mDeferredActions.push_front( 1307 new SimpleAction( 1308 &NuPlayer::performScanSources)); 1309 } 1310 1311 if (formatChange /* not seamless */) { 1312 // must change decoder 1313 flushDecoder(audio, /* needShutdown = */ true); 1314 } else if (timeChange) { 1315 // need to flush 1316 flushDecoder(audio, /* needShutdown = */ false, newFormat); 1317 err = OK; 1318 } else if (seamlessFormatChange) { 1319 // reuse existing decoder and don't flush 1320 updateDecoderFormatWithoutFlush(audio, newFormat); 1321 err = OK; 1322 } else { 1323 // This stream is unaffected by the discontinuity 1324 return -EWOULDBLOCK; 1325 } 1326 } 1327 1328 reply->setInt32("err", err); 1329 reply->post(); 1330 return OK; 1331 } 1332 1333 if (!audio) { 1334 ++mNumFramesTotal; 1335 } 1336 1337 dropAccessUnit = false; 1338 if (!audio 1339 && !(mSourceFlags & Source::FLAG_SECURE) 1340 && mRenderer->getVideoLateByUs() > 100000ll 1341 && mVideoIsAVC 1342 && !IsAVCReferenceFrame(accessUnit)) { 1343 dropAccessUnit = true; 1344 ++mNumFramesDropped; 1345 } 1346 1347 size_t smallSize = accessUnit->size(); 1348 needMoreData = false; 1349 if (doBufferAggregation && (mAggregateBuffer == NULL) 1350 // Don't bother if only room for a few small buffers. 1351 && (smallSize < (kAggregateBufferSizeBytes / 3))) { 1352 // Create a larger buffer for combining smaller buffers from the extractor. 1353 mAggregateBuffer = new ABuffer(kAggregateBufferSizeBytes); 1354 mAggregateBuffer->setRange(0, 0); // start empty 1355 } 1356 1357 if (doBufferAggregation && (mAggregateBuffer != NULL)) { 1358 int64_t timeUs; 1359 int64_t dummy; 1360 bool smallTimestampValid = accessUnit->meta()->findInt64("timeUs", &timeUs); 1361 bool bigTimestampValid = mAggregateBuffer->meta()->findInt64("timeUs", &dummy); 1362 // Will the smaller buffer fit? 1363 size_t bigSize = mAggregateBuffer->size(); 1364 size_t roomLeft = mAggregateBuffer->capacity() - bigSize; 1365 // Should we save this small buffer for the next big buffer? 1366 // If the first small buffer did not have a timestamp then save 1367 // any buffer that does have a timestamp until the next big buffer. 1368 if ((smallSize > roomLeft) 1369 || (!bigTimestampValid && (bigSize > 0) && smallTimestampValid)) { 1370 mPendingAudioErr = err; 1371 mPendingAudioAccessUnit = accessUnit; 1372 accessUnit.clear(); 1373 } else { 1374 // Grab time from first small buffer if available. 1375 if ((bigSize == 0) && smallTimestampValid) { 1376 mAggregateBuffer->meta()->setInt64("timeUs", timeUs); 1377 } 1378 // Append small buffer to the bigger buffer. 1379 memcpy(mAggregateBuffer->base() + bigSize, accessUnit->data(), smallSize); 1380 bigSize += smallSize; 1381 mAggregateBuffer->setRange(0, bigSize); 1382 1383 // Keep looping until we run out of room in the mAggregateBuffer. 1384 needMoreData = true; 1385 1386 ALOGV("feedDecoderInputData() smallSize = %zu, bigSize = %zu, capacity = %zu", 1387 smallSize, bigSize, mAggregateBuffer->capacity()); 1388 } 1389 } 1390 } while (dropAccessUnit || needMoreData); 1391 1392 // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video"); 1393 1394#if 0 1395 int64_t mediaTimeUs; 1396 CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); 1397 ALOGV("feeding %s input buffer at media time %.2f secs", 1398 audio ? "audio" : "video", 1399 mediaTimeUs / 1E6); 1400#endif 1401 1402 if (!audio) { 1403 mCCDecoder->decode(accessUnit); 1404 } 1405 1406 if (doBufferAggregation && (mAggregateBuffer != NULL)) { 1407 ALOGV("feedDecoderInputData() reply with aggregated buffer, %zu", 1408 mAggregateBuffer->size()); 1409 reply->setBuffer("buffer", mAggregateBuffer); 1410 mAggregateBuffer.clear(); 1411 } else { 1412 reply->setBuffer("buffer", accessUnit); 1413 } 1414 1415 reply->post(); 1416 1417 return OK; 1418} 1419 1420void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) { 1421 // ALOGV("renderBuffer %s", audio ? "audio" : "video"); 1422 1423 if ((audio && mFlushingAudio != NONE) 1424 || (!audio && mFlushingVideo != NONE)) { 1425 // We're currently attempting to flush the decoder, in order 1426 // to complete this, the decoder wants all its buffers back, 1427 // so we don't want any output buffers it sent us (from before 1428 // we initiated the flush) to be stuck in the renderer's queue. 1429 1430 ALOGV("we're still flushing the %s decoder, sending its output buffer" 1431 " right back.", audio ? "audio" : "video"); 1432 1433 return; 1434 } 1435 1436 int64_t mediaTimeUs; 1437 CHECK(msg->findInt64("timeUs", &mediaTimeUs)); 1438 1439 if (!audio && mCCDecoder->isSelected()) { 1440 mCCDecoder->display(mediaTimeUs); 1441 } 1442} 1443 1444void NuPlayer::updateVideoSize( 1445 const sp<AMessage> &inputFormat, 1446 const sp<AMessage> &outputFormat) { 1447 if (inputFormat == NULL) { 1448 ALOGW("Unknown video size, reporting 0x0!"); 1449 notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0); 1450 return; 1451 } 1452 1453 int32_t displayWidth, displayHeight; 1454 int32_t cropLeft, cropTop, cropRight, cropBottom; 1455 1456 if (outputFormat != NULL) { 1457 int32_t width, height; 1458 CHECK(outputFormat->findInt32("width", &width)); 1459 CHECK(outputFormat->findInt32("height", &height)); 1460 1461 int32_t cropLeft, cropTop, cropRight, cropBottom; 1462 CHECK(outputFormat->findRect( 1463 "crop", 1464 &cropLeft, &cropTop, &cropRight, &cropBottom)); 1465 1466 displayWidth = cropRight - cropLeft + 1; 1467 displayHeight = cropBottom - cropTop + 1; 1468 1469 ALOGV("Video output format changed to %d x %d " 1470 "(crop: %d x %d @ (%d, %d))", 1471 width, height, 1472 displayWidth, 1473 displayHeight, 1474 cropLeft, cropTop); 1475 } else { 1476 CHECK(inputFormat->findInt32("width", &displayWidth)); 1477 CHECK(inputFormat->findInt32("height", &displayHeight)); 1478 1479 ALOGV("Video input format %d x %d", displayWidth, displayHeight); 1480 } 1481 1482 // Take into account sample aspect ratio if necessary: 1483 int32_t sarWidth, sarHeight; 1484 if (inputFormat->findInt32("sar-width", &sarWidth) 1485 && inputFormat->findInt32("sar-height", &sarHeight)) { 1486 ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight); 1487 1488 displayWidth = (displayWidth * sarWidth) / sarHeight; 1489 1490 ALOGV("display dimensions %d x %d", displayWidth, displayHeight); 1491 } 1492 1493 int32_t rotationDegrees; 1494 if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) { 1495 rotationDegrees = 0; 1496 } 1497 1498 if (rotationDegrees == 90 || rotationDegrees == 270) { 1499 int32_t tmp = displayWidth; 1500 displayWidth = displayHeight; 1501 displayHeight = tmp; 1502 } 1503 1504 notifyListener( 1505 MEDIA_SET_VIDEO_SIZE, 1506 displayWidth, 1507 displayHeight); 1508} 1509 1510void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) { 1511 if (mDriver == NULL) { 1512 return; 1513 } 1514 1515 sp<NuPlayerDriver> driver = mDriver.promote(); 1516 1517 if (driver == NULL) { 1518 return; 1519 } 1520 1521 driver->notifyListener(msg, ext1, ext2, in); 1522} 1523 1524void NuPlayer::flushDecoder( 1525 bool audio, bool needShutdown, const sp<AMessage> &newFormat) { 1526 ALOGV("[%s] flushDecoder needShutdown=%d", 1527 audio ? "audio" : "video", needShutdown); 1528 1529 const sp<Decoder> &decoder = getDecoder(audio); 1530 if (decoder == NULL) { 1531 ALOGI("flushDecoder %s without decoder present", 1532 audio ? "audio" : "video"); 1533 return; 1534 } 1535 1536 // Make sure we don't continue to scan sources until we finish flushing. 1537 ++mScanSourcesGeneration; 1538 mScanSourcesPending = false; 1539 1540 decoder->signalFlush(newFormat); 1541 1542 FlushStatus newStatus = 1543 needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER; 1544 1545 mFlushComplete[audio][false /* isDecoder */] = false; 1546 mFlushComplete[audio][true /* isDecoder */] = false; 1547 if (audio) { 1548 ALOGE_IF(mFlushingAudio != NONE, 1549 "audio flushDecoder() is called in state %d", mFlushingAudio); 1550 mFlushingAudio = newStatus; 1551 } else { 1552 ALOGE_IF(mFlushingVideo != NONE, 1553 "video flushDecoder() is called in state %d", mFlushingVideo); 1554 mFlushingVideo = newStatus; 1555 1556 if (mCCDecoder != NULL) { 1557 mCCDecoder->flush(); 1558 } 1559 } 1560} 1561 1562void NuPlayer::updateDecoderFormatWithoutFlush( 1563 bool audio, const sp<AMessage> &format) { 1564 ALOGV("[%s] updateDecoderFormatWithoutFlush", audio ? "audio" : "video"); 1565 1566 const sp<Decoder> &decoder = getDecoder(audio); 1567 if (decoder == NULL) { 1568 ALOGI("updateDecoderFormatWithoutFlush %s without decoder present", 1569 audio ? "audio" : "video"); 1570 return; 1571 } 1572 1573 decoder->signalUpdateFormat(format); 1574} 1575 1576void NuPlayer::queueDecoderShutdown( 1577 bool audio, bool video, const sp<AMessage> &reply) { 1578 ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video); 1579 1580 mDeferredActions.push_back( 1581 new FlushDecoderAction( 1582 audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE, 1583 video ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE)); 1584 1585 mDeferredActions.push_back( 1586 new SimpleAction(&NuPlayer::performScanSources)); 1587 1588 mDeferredActions.push_back(new PostMessageAction(reply)); 1589 1590 processDeferredActions(); 1591} 1592 1593status_t NuPlayer::setVideoScalingMode(int32_t mode) { 1594 mVideoScalingMode = mode; 1595 if (mNativeWindow != NULL) { 1596 status_t ret = native_window_set_scaling_mode( 1597 mNativeWindow->getNativeWindow().get(), mVideoScalingMode); 1598 if (ret != OK) { 1599 ALOGE("Failed to set scaling mode (%d): %s", 1600 -ret, strerror(-ret)); 1601 return ret; 1602 } 1603 } 1604 return OK; 1605} 1606 1607status_t NuPlayer::getTrackInfo(Parcel* reply) const { 1608 sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id()); 1609 msg->setPointer("reply", reply); 1610 1611 sp<AMessage> response; 1612 status_t err = msg->postAndAwaitResponse(&response); 1613 return err; 1614} 1615 1616status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const { 1617 sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id()); 1618 msg->setPointer("reply", reply); 1619 msg->setInt32("type", type); 1620 1621 sp<AMessage> response; 1622 status_t err = msg->postAndAwaitResponse(&response); 1623 if (err == OK && response != NULL) { 1624 CHECK(response->findInt32("err", &err)); 1625 } 1626 return err; 1627} 1628 1629status_t NuPlayer::selectTrack(size_t trackIndex, bool select, int64_t timeUs) { 1630 sp<AMessage> msg = new AMessage(kWhatSelectTrack, id()); 1631 msg->setSize("trackIndex", trackIndex); 1632 msg->setInt32("select", select); 1633 msg->setInt64("timeUs", timeUs); 1634 1635 sp<AMessage> response; 1636 status_t err = msg->postAndAwaitResponse(&response); 1637 1638 if (err != OK) { 1639 return err; 1640 } 1641 1642 if (!response->findInt32("err", &err)) { 1643 err = OK; 1644 } 1645 1646 return err; 1647} 1648 1649status_t NuPlayer::getCurrentPosition(int64_t *mediaUs) { 1650 sp<Renderer> renderer = mRenderer; 1651 if (renderer == NULL) { 1652 return NO_INIT; 1653 } 1654 1655 return renderer->getCurrentPosition(mediaUs); 1656} 1657 1658void NuPlayer::getStats(int64_t *numFramesTotal, int64_t *numFramesDropped) { 1659 *numFramesTotal = mNumFramesTotal; 1660 *numFramesDropped = mNumFramesDropped; 1661} 1662 1663sp<MetaData> NuPlayer::getFileMeta() { 1664 return mSource->getFileFormatMeta(); 1665} 1666 1667void NuPlayer::schedulePollDuration() { 1668 sp<AMessage> msg = new AMessage(kWhatPollDuration, id()); 1669 msg->setInt32("generation", mPollDurationGeneration); 1670 msg->post(); 1671} 1672 1673void NuPlayer::cancelPollDuration() { 1674 ++mPollDurationGeneration; 1675} 1676 1677void NuPlayer::processDeferredActions() { 1678 while (!mDeferredActions.empty()) { 1679 // We won't execute any deferred actions until we're no longer in 1680 // an intermediate state, i.e. one more more decoders are currently 1681 // flushing or shutting down. 1682 1683 if (mFlushingAudio != NONE || mFlushingVideo != NONE) { 1684 // We're currently flushing, postpone the reset until that's 1685 // completed. 1686 1687 ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d", 1688 mFlushingAudio, mFlushingVideo); 1689 1690 break; 1691 } 1692 1693 sp<Action> action = *mDeferredActions.begin(); 1694 mDeferredActions.erase(mDeferredActions.begin()); 1695 1696 action->execute(this); 1697 } 1698} 1699 1700void NuPlayer::performSeek(int64_t seekTimeUs, bool needNotify) { 1701 ALOGV("performSeek seekTimeUs=%lld us (%.2f secs), needNotify(%d)", 1702 seekTimeUs, 1703 seekTimeUs / 1E6, 1704 needNotify); 1705 1706 if (mSource == NULL) { 1707 // This happens when reset occurs right before the loop mode 1708 // asynchronously seeks to the start of the stream. 1709 LOG_ALWAYS_FATAL_IF(mAudioDecoder != NULL || mVideoDecoder != NULL, 1710 "mSource is NULL and decoders not NULL audio(%p) video(%p)", 1711 mAudioDecoder.get(), mVideoDecoder.get()); 1712 return; 1713 } 1714 mSource->seekTo(seekTimeUs); 1715 ++mTimedTextGeneration; 1716 1717 if (mDriver != NULL) { 1718 sp<NuPlayerDriver> driver = mDriver.promote(); 1719 if (driver != NULL) { 1720 if (needNotify) { 1721 driver->notifySeekComplete(); 1722 } 1723 } 1724 } 1725 1726 // everything's flushed, continue playback. 1727} 1728 1729void NuPlayer::performDecoderFlush(FlushCommand audio, FlushCommand video) { 1730 ALOGV("performDecoderFlush audio=%d, video=%d", audio, video); 1731 1732 if ((audio == FLUSH_CMD_NONE || mAudioDecoder == NULL) 1733 && (video == FLUSH_CMD_NONE || mVideoDecoder == NULL)) { 1734 return; 1735 } 1736 1737 mTimeDiscontinuityPending = true; 1738 1739 if (audio != FLUSH_CMD_NONE && mAudioDecoder != NULL) { 1740 flushDecoder(true /* audio */, (audio == FLUSH_CMD_SHUTDOWN)); 1741 } 1742 1743 if (video != FLUSH_CMD_NONE && mVideoDecoder != NULL) { 1744 flushDecoder(false /* audio */, (video == FLUSH_CMD_SHUTDOWN)); 1745 } 1746} 1747 1748void NuPlayer::performReset() { 1749 ALOGV("performReset"); 1750 1751 CHECK(mAudioDecoder == NULL); 1752 CHECK(mVideoDecoder == NULL); 1753 1754 cancelPollDuration(); 1755 1756 ++mScanSourcesGeneration; 1757 mScanSourcesPending = false; 1758 1759 if (mRendererLooper != NULL) { 1760 if (mRenderer != NULL) { 1761 mRendererLooper->unregisterHandler(mRenderer->id()); 1762 } 1763 mRendererLooper->stop(); 1764 mRendererLooper.clear(); 1765 } 1766 mRenderer.clear(); 1767 ++mRendererGeneration; 1768 1769 if (mSource != NULL) { 1770 mSource->stop(); 1771 1772 mSource.clear(); 1773 } 1774 1775 if (mDriver != NULL) { 1776 sp<NuPlayerDriver> driver = mDriver.promote(); 1777 if (driver != NULL) { 1778 driver->notifyResetComplete(); 1779 } 1780 } 1781 1782 mStarted = false; 1783} 1784 1785void NuPlayer::performScanSources() { 1786 ALOGV("performScanSources"); 1787 1788 if (!mStarted) { 1789 return; 1790 } 1791 1792 if (mAudioDecoder == NULL || mVideoDecoder == NULL) { 1793 postScanSources(); 1794 } 1795} 1796 1797void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) { 1798 ALOGV("performSetSurface"); 1799 1800 mNativeWindow = wrapper; 1801 1802 // XXX - ignore error from setVideoScalingMode for now 1803 setVideoScalingMode(mVideoScalingMode); 1804 1805 if (mDriver != NULL) { 1806 sp<NuPlayerDriver> driver = mDriver.promote(); 1807 if (driver != NULL) { 1808 driver->notifySetSurfaceComplete(); 1809 } 1810 } 1811} 1812 1813void NuPlayer::onSourceNotify(const sp<AMessage> &msg) { 1814 int32_t what; 1815 CHECK(msg->findInt32("what", &what)); 1816 1817 switch (what) { 1818 case Source::kWhatPrepared: 1819 { 1820 if (mSource == NULL) { 1821 // This is a stale notification from a source that was 1822 // asynchronously preparing when the client called reset(). 1823 // We handled the reset, the source is gone. 1824 break; 1825 } 1826 1827 int32_t err; 1828 CHECK(msg->findInt32("err", &err)); 1829 1830 sp<NuPlayerDriver> driver = mDriver.promote(); 1831 if (driver != NULL) { 1832 // notify duration first, so that it's definitely set when 1833 // the app received the "prepare complete" callback. 1834 int64_t durationUs; 1835 if (mSource->getDuration(&durationUs) == OK) { 1836 driver->notifyDuration(durationUs); 1837 } 1838 driver->notifyPrepareCompleted(err); 1839 } 1840 1841 break; 1842 } 1843 1844 case Source::kWhatFlagsChanged: 1845 { 1846 uint32_t flags; 1847 CHECK(msg->findInt32("flags", (int32_t *)&flags)); 1848 1849 sp<NuPlayerDriver> driver = mDriver.promote(); 1850 if (driver != NULL) { 1851 driver->notifyFlagsChanged(flags); 1852 } 1853 1854 if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION) 1855 && (!(flags & Source::FLAG_DYNAMIC_DURATION))) { 1856 cancelPollDuration(); 1857 } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION) 1858 && (flags & Source::FLAG_DYNAMIC_DURATION) 1859 && (mAudioDecoder != NULL || mVideoDecoder != NULL)) { 1860 schedulePollDuration(); 1861 } 1862 1863 mSourceFlags = flags; 1864 break; 1865 } 1866 1867 case Source::kWhatVideoSizeChanged: 1868 { 1869 sp<AMessage> format; 1870 CHECK(msg->findMessage("format", &format)); 1871 1872 updateVideoSize(format); 1873 break; 1874 } 1875 1876 case Source::kWhatBufferingUpdate: 1877 { 1878 int32_t percentage; 1879 CHECK(msg->findInt32("percentage", &percentage)); 1880 1881 notifyListener(MEDIA_BUFFERING_UPDATE, percentage, 0); 1882 break; 1883 } 1884 1885 case Source::kWhatBufferingStart: 1886 { 1887 notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0); 1888 break; 1889 } 1890 1891 case Source::kWhatBufferingEnd: 1892 { 1893 notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0); 1894 break; 1895 } 1896 1897 case Source::kWhatSubtitleData: 1898 { 1899 sp<ABuffer> buffer; 1900 CHECK(msg->findBuffer("buffer", &buffer)); 1901 1902 sendSubtitleData(buffer, 0 /* baseIndex */); 1903 break; 1904 } 1905 1906 case Source::kWhatTimedTextData: 1907 { 1908 int32_t generation; 1909 if (msg->findInt32("generation", &generation) 1910 && generation != mTimedTextGeneration) { 1911 break; 1912 } 1913 1914 sp<ABuffer> buffer; 1915 CHECK(msg->findBuffer("buffer", &buffer)); 1916 1917 sp<NuPlayerDriver> driver = mDriver.promote(); 1918 if (driver == NULL) { 1919 break; 1920 } 1921 1922 int posMs; 1923 int64_t timeUs, posUs; 1924 driver->getCurrentPosition(&posMs); 1925 posUs = posMs * 1000; 1926 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 1927 1928 if (posUs < timeUs) { 1929 if (!msg->findInt32("generation", &generation)) { 1930 msg->setInt32("generation", mTimedTextGeneration); 1931 } 1932 msg->post(timeUs - posUs); 1933 } else { 1934 sendTimedTextData(buffer); 1935 } 1936 break; 1937 } 1938 1939 case Source::kWhatQueueDecoderShutdown: 1940 { 1941 int32_t audio, video; 1942 CHECK(msg->findInt32("audio", &audio)); 1943 CHECK(msg->findInt32("video", &video)); 1944 1945 sp<AMessage> reply; 1946 CHECK(msg->findMessage("reply", &reply)); 1947 1948 queueDecoderShutdown(audio, video, reply); 1949 break; 1950 } 1951 1952 case Source::kWhatDrmNoLicense: 1953 { 1954 notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, ERROR_DRM_NO_LICENSE); 1955 break; 1956 } 1957 1958 default: 1959 TRESPASS(); 1960 } 1961} 1962 1963void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) { 1964 int32_t what; 1965 CHECK(msg->findInt32("what", &what)); 1966 1967 switch (what) { 1968 case NuPlayer::CCDecoder::kWhatClosedCaptionData: 1969 { 1970 sp<ABuffer> buffer; 1971 CHECK(msg->findBuffer("buffer", &buffer)); 1972 1973 size_t inbandTracks = 0; 1974 if (mSource != NULL) { 1975 inbandTracks = mSource->getTrackCount(); 1976 } 1977 1978 sendSubtitleData(buffer, inbandTracks); 1979 break; 1980 } 1981 1982 case NuPlayer::CCDecoder::kWhatTrackAdded: 1983 { 1984 notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0); 1985 1986 break; 1987 } 1988 1989 default: 1990 TRESPASS(); 1991 } 1992 1993 1994} 1995 1996void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) { 1997 int32_t trackIndex; 1998 int64_t timeUs, durationUs; 1999 CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex)); 2000 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 2001 CHECK(buffer->meta()->findInt64("durationUs", &durationUs)); 2002 2003 Parcel in; 2004 in.writeInt32(trackIndex + baseIndex); 2005 in.writeInt64(timeUs); 2006 in.writeInt64(durationUs); 2007 in.writeInt32(buffer->size()); 2008 in.writeInt32(buffer->size()); 2009 in.write(buffer->data(), buffer->size()); 2010 2011 notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in); 2012} 2013 2014void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) { 2015 const void *data; 2016 size_t size = 0; 2017 int64_t timeUs; 2018 int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS; 2019 2020 AString mime; 2021 CHECK(buffer->meta()->findString("mime", &mime)); 2022 CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0); 2023 2024 data = buffer->data(); 2025 size = buffer->size(); 2026 2027 Parcel parcel; 2028 if (size > 0) { 2029 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 2030 flag |= TextDescriptions::IN_BAND_TEXT_3GPP; 2031 TextDescriptions::getParcelOfDescriptions( 2032 (const uint8_t *)data, size, flag, timeUs / 1000, &parcel); 2033 } 2034 2035 if ((parcel.dataSize() > 0)) { 2036 notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel); 2037 } else { // send an empty timed text 2038 notifyListener(MEDIA_TIMED_TEXT, 0, 0); 2039 } 2040} 2041//////////////////////////////////////////////////////////////////////////////// 2042 2043sp<AMessage> NuPlayer::Source::getFormat(bool audio) { 2044 sp<MetaData> meta = getFormatMeta(audio); 2045 2046 if (meta == NULL) { 2047 return NULL; 2048 } 2049 2050 sp<AMessage> msg = new AMessage; 2051 2052 if(convertMetaDataToMessage(meta, &msg) == OK) { 2053 return msg; 2054 } 2055 return NULL; 2056} 2057 2058void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) { 2059 sp<AMessage> notify = dupNotify(); 2060 notify->setInt32("what", kWhatFlagsChanged); 2061 notify->setInt32("flags", flags); 2062 notify->post(); 2063} 2064 2065void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) { 2066 sp<AMessage> notify = dupNotify(); 2067 notify->setInt32("what", kWhatVideoSizeChanged); 2068 notify->setMessage("format", format); 2069 notify->post(); 2070} 2071 2072void NuPlayer::Source::notifyPrepared(status_t err) { 2073 sp<AMessage> notify = dupNotify(); 2074 notify->setInt32("what", kWhatPrepared); 2075 notify->setInt32("err", err); 2076 notify->post(); 2077} 2078 2079void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) { 2080 TRESPASS(); 2081} 2082 2083} // namespace android 2084