NuPlayerDecoder.cpp revision c387f2b719a1a26c8306f77d79cc9a6f26b36813
1/* 2 * Copyright 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "NuPlayerDecoder" 19#include <utils/Log.h> 20#include <inttypes.h> 21 22#include <algorithm> 23 24#include "NuPlayerCCDecoder.h" 25#include "NuPlayerDecoder.h" 26#include "NuPlayerDrm.h" 27#include "NuPlayerRenderer.h" 28#include "NuPlayerSource.h" 29 30#include <cutils/properties.h> 31#include <media/ICrypto.h> 32#include <media/MediaCodecBuffer.h> 33#include <media/stagefright/foundation/ABuffer.h> 34#include <media/stagefright/foundation/ADebug.h> 35#include <media/stagefright/foundation/AMessage.h> 36#include <media/stagefright/MediaBuffer.h> 37#include <media/stagefright/MediaCodec.h> 38#include <media/stagefright/MediaDefs.h> 39#include <media/stagefright/MediaErrors.h> 40#include <media/stagefright/SurfaceUtils.h> 41#include <gui/Surface.h> 42 43#include "avc_utils.h" 44#include "ATSParser.h" 45 46namespace android { 47 48static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display 49 50// The default total video frame rate of a stream when that info is not available from 51// the source. 52static float kDefaultVideoFrameRateTotal = 30.f; 53 54static inline bool getAudioDeepBufferSetting() { 55 return property_get_bool("media.stagefright.audio.deep", false /* default_value */); 56} 57 58NuPlayer::Decoder::Decoder( 59 const sp<AMessage> ¬ify, 60 const sp<Source> &source, 61 pid_t pid, 62 uid_t uid, 63 const sp<Renderer> &renderer, 64 const sp<Surface> &surface, 65 const sp<CCDecoder> &ccDecoder) 66 : DecoderBase(notify), 67 mSurface(surface), 68 mSource(source), 69 mRenderer(renderer), 70 mCCDecoder(ccDecoder), 71 mPid(pid), 72 mUid(uid), 73 mSkipRenderingUntilMediaTimeUs(-1ll), 74 mNumFramesTotal(0ll), 75 mNumInputFramesDropped(0ll), 76 mNumOutputFramesDropped(0ll), 77 mVideoWidth(0), 78 mVideoHeight(0), 79 mIsAudio(true), 80 mIsVideoAVC(false), 81 mIsSecure(false), 82 mFormatChangePending(false), 83 mTimeChangePending(false), 84 mFrameRateTotal(kDefaultVideoFrameRateTotal), 85 mPlaybackSpeed(1.0f), 86 mNumVideoTemporalLayerTotal(1), // decode all layers 87 mNumVideoTemporalLayerAllowed(1), 88 mCurrentMaxVideoTemporalLayerId(0), 89 mResumePending(false), 90 mComponentName("decoder") { 91 mCodecLooper = new ALooper; 92 mCodecLooper->setName("NPDecoder-CL"); 93 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 94 mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal; 95} 96 97NuPlayer::Decoder::~Decoder() { 98 mCodec->release(); 99 releaseAndResetMediaBuffers(); 100} 101 102sp<AMessage> NuPlayer::Decoder::getStats() const { 103 mStats->setInt64("frames-total", mNumFramesTotal); 104 mStats->setInt64("frames-dropped-input", mNumInputFramesDropped); 105 mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped); 106 return mStats; 107} 108 109status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) { 110 if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) { 111 return BAD_VALUE; 112 } 113 114 sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this); 115 116 msg->setObject("surface", surface); 117 sp<AMessage> response; 118 status_t err = msg->postAndAwaitResponse(&response); 119 if (err == OK && response != NULL) { 120 CHECK(response->findInt32("err", &err)); 121 } 122 return err; 123} 124 125void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) { 126 ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str()); 127 128 switch (msg->what()) { 129 case kWhatCodecNotify: 130 { 131 int32_t cbID; 132 CHECK(msg->findInt32("callbackID", &cbID)); 133 134 ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d", 135 mIsAudio ? "audio" : "video", cbID, mPaused); 136 137 if (mPaused) { 138 break; 139 } 140 141 switch (cbID) { 142 case MediaCodec::CB_INPUT_AVAILABLE: 143 { 144 int32_t index; 145 CHECK(msg->findInt32("index", &index)); 146 147 handleAnInputBuffer(index); 148 break; 149 } 150 151 case MediaCodec::CB_OUTPUT_AVAILABLE: 152 { 153 int32_t index; 154 size_t offset; 155 size_t size; 156 int64_t timeUs; 157 int32_t flags; 158 159 CHECK(msg->findInt32("index", &index)); 160 CHECK(msg->findSize("offset", &offset)); 161 CHECK(msg->findSize("size", &size)); 162 CHECK(msg->findInt64("timeUs", &timeUs)); 163 CHECK(msg->findInt32("flags", &flags)); 164 165 handleAnOutputBuffer(index, offset, size, timeUs, flags); 166 break; 167 } 168 169 case MediaCodec::CB_OUTPUT_FORMAT_CHANGED: 170 { 171 sp<AMessage> format; 172 CHECK(msg->findMessage("format", &format)); 173 174 handleOutputFormatChange(format); 175 break; 176 } 177 178 case MediaCodec::CB_ERROR: 179 { 180 status_t err; 181 CHECK(msg->findInt32("err", &err)); 182 ALOGE("Decoder (%s) reported error : 0x%x", 183 mIsAudio ? "audio" : "video", err); 184 185 handleError(err); 186 break; 187 } 188 189 default: 190 { 191 TRESPASS(); 192 break; 193 } 194 } 195 196 break; 197 } 198 199 case kWhatRenderBuffer: 200 { 201 if (!isStaleReply(msg)) { 202 onRenderBuffer(msg); 203 } 204 break; 205 } 206 207 case kWhatAudioOutputFormatChanged: 208 { 209 if (!isStaleReply(msg)) { 210 status_t err; 211 if (msg->findInt32("err", &err) && err != OK) { 212 ALOGE("Renderer reported 0x%x when changing audio output format", err); 213 handleError(err); 214 } 215 } 216 break; 217 } 218 219 case kWhatSetVideoSurface: 220 { 221 sp<AReplyToken> replyID; 222 CHECK(msg->senderAwaitsResponse(&replyID)); 223 224 sp<RefBase> obj; 225 CHECK(msg->findObject("surface", &obj)); 226 sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null 227 int32_t err = INVALID_OPERATION; 228 // NOTE: in practice mSurface is always non-null, but checking here for completeness 229 if (mCodec != NULL && mSurface != NULL) { 230 // TODO: once AwesomePlayer is removed, remove this automatic connecting 231 // to the surface by MediaPlayerService. 232 // 233 // at this point MediaPlayerService::client has already connected to the 234 // surface, which MediaCodec does not expect 235 err = nativeWindowDisconnect(surface.get(), "kWhatSetVideoSurface(surface)"); 236 if (err == OK) { 237 err = mCodec->setSurface(surface); 238 ALOGI_IF(err, "codec setSurface returned: %d", err); 239 if (err == OK) { 240 // reconnect to the old surface as MPS::Client will expect to 241 // be able to disconnect from it. 242 (void)nativeWindowConnect(mSurface.get(), "kWhatSetVideoSurface(mSurface)"); 243 mSurface = surface; 244 } 245 } 246 if (err != OK) { 247 // reconnect to the new surface on error as MPS::Client will expect to 248 // be able to disconnect from it. 249 (void)nativeWindowConnect(surface.get(), "kWhatSetVideoSurface(err)"); 250 } 251 } 252 253 sp<AMessage> response = new AMessage; 254 response->setInt32("err", err); 255 response->postReply(replyID); 256 break; 257 } 258 259 case kWhatDrmReleaseCrypto: 260 { 261 ALOGV("kWhatDrmReleaseCrypto"); 262 onReleaseCrypto(msg); 263 break; 264 } 265 266 default: 267 DecoderBase::onMessageReceived(msg); 268 break; 269 } 270} 271 272void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) { 273 CHECK(mCodec == NULL); 274 275 mFormatChangePending = false; 276 mTimeChangePending = false; 277 278 ++mBufferGeneration; 279 280 AString mime; 281 CHECK(format->findString("mime", &mime)); 282 283 mIsAudio = !strncasecmp("audio/", mime.c_str(), 6); 284 mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str()); 285 286 mComponentName = mime; 287 mComponentName.append(" decoder"); 288 ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get()); 289 290 mCodec = MediaCodec::CreateByType( 291 mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid); 292 int32_t secure = 0; 293 if (format->findInt32("secure", &secure) && secure != 0) { 294 if (mCodec != NULL) { 295 mCodec->getName(&mComponentName); 296 mComponentName.append(".secure"); 297 mCodec->release(); 298 ALOGI("[%s] creating", mComponentName.c_str()); 299 mCodec = MediaCodec::CreateByComponentName( 300 mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid); 301 } 302 } 303 if (mCodec == NULL) { 304 ALOGE("Failed to create %s%s decoder", 305 (secure ? "secure " : ""), mime.c_str()); 306 handleError(UNKNOWN_ERROR); 307 return; 308 } 309 mIsSecure = secure; 310 311 mCodec->getName(&mComponentName); 312 313 status_t err; 314 if (mSurface != NULL) { 315 // disconnect from surface as MediaCodec will reconnect 316 err = nativeWindowDisconnect(mSurface.get(), "onConfigure"); 317 // We treat this as a warning, as this is a preparatory step. 318 // Codec will try to connect to the surface, which is where 319 // any error signaling will occur. 320 ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err); 321 } 322 323 // Modular DRM 324 void *pCrypto; 325 if (!format->findPointer("crypto", &pCrypto)) { 326 pCrypto = NULL; 327 } 328 sp<ICrypto> crypto = (ICrypto*)pCrypto; 329 ALOGV("onConfigure mCrypto: %p (%d) mIsSecure: %d", 330 crypto.get(), (crypto != NULL ? crypto->getStrongCount() : 0), mIsSecure); 331 332 err = mCodec->configure( 333 format, mSurface, crypto, 0 /* flags */); 334 335 if (err != OK) { 336 ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err); 337 mCodec->release(); 338 mCodec.clear(); 339 handleError(err); 340 return; 341 } 342 rememberCodecSpecificData(format); 343 344 // the following should work in configured state 345 CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat)); 346 CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat)); 347 348 mStats->setString("mime", mime.c_str()); 349 mStats->setString("component-name", mComponentName.c_str()); 350 351 if (!mIsAudio) { 352 int32_t width, height; 353 if (mOutputFormat->findInt32("width", &width) 354 && mOutputFormat->findInt32("height", &height)) { 355 mStats->setInt32("width", width); 356 mStats->setInt32("height", height); 357 } 358 } 359 360 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this); 361 mCodec->setCallback(reply); 362 363 err = mCodec->start(); 364 if (err != OK) { 365 ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err); 366 mCodec->release(); 367 mCodec.clear(); 368 handleError(err); 369 return; 370 } 371 372 releaseAndResetMediaBuffers(); 373 374 mPaused = false; 375 mResumePending = false; 376} 377 378void NuPlayer::Decoder::onSetParameters(const sp<AMessage> ¶ms) { 379 bool needAdjustLayers = false; 380 float frameRateTotal; 381 if (params->findFloat("frame-rate-total", &frameRateTotal) 382 && mFrameRateTotal != frameRateTotal) { 383 needAdjustLayers = true; 384 mFrameRateTotal = frameRateTotal; 385 } 386 387 int32_t numVideoTemporalLayerTotal; 388 if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal) 389 && numVideoTemporalLayerTotal >= 0 390 && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers 391 && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) { 392 needAdjustLayers = true; 393 mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1); 394 } 395 396 if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) { 397 // TODO: For now, layer fps is calculated for some specific architectures. 398 // But it really should be extracted from the stream. 399 mVideoTemporalLayerAggregateFps[0] = 400 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1)); 401 for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) { 402 mVideoTemporalLayerAggregateFps[i] = 403 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i)) 404 + mVideoTemporalLayerAggregateFps[i - 1]; 405 } 406 } 407 408 float playbackSpeed; 409 if (params->findFloat("playback-speed", &playbackSpeed) 410 && mPlaybackSpeed != playbackSpeed) { 411 needAdjustLayers = true; 412 mPlaybackSpeed = playbackSpeed; 413 } 414 415 if (needAdjustLayers) { 416 float decodeFrameRate = mFrameRateTotal; 417 // enable temporal layering optimization only if we know the layering depth 418 if (mNumVideoTemporalLayerTotal > 1) { 419 int32_t layerId; 420 for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) { 421 if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed 422 >= kDisplayRefreshingRate * 0.9) { 423 break; 424 } 425 } 426 mNumVideoTemporalLayerAllowed = layerId + 1; 427 decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId]; 428 } 429 ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g", 430 mNumVideoTemporalLayerAllowed, decodeFrameRate); 431 432 if (mCodec == NULL) { 433 ALOGW("onSetParameters called before codec is created."); 434 return; 435 } 436 437 sp<AMessage> codecParams = new AMessage(); 438 codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed); 439 mCodec->setParameters(codecParams); 440 } 441} 442 443void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) { 444 mRenderer = renderer; 445} 446 447void NuPlayer::Decoder::onResume(bool notifyComplete) { 448 mPaused = false; 449 450 if (notifyComplete) { 451 mResumePending = true; 452 } 453 mCodec->start(); 454} 455 456void NuPlayer::Decoder::doFlush(bool notifyComplete) { 457 if (mCCDecoder != NULL) { 458 mCCDecoder->flush(); 459 } 460 461 if (mRenderer != NULL) { 462 mRenderer->flush(mIsAudio, notifyComplete); 463 mRenderer->signalTimeDiscontinuity(); 464 } 465 466 status_t err = OK; 467 if (mCodec != NULL) { 468 err = mCodec->flush(); 469 mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator 470 ++mBufferGeneration; 471 } 472 473 if (err != OK) { 474 ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err); 475 handleError(err); 476 // finish with posting kWhatFlushCompleted. 477 // we attempt to release the buffers even if flush fails. 478 } 479 releaseAndResetMediaBuffers(); 480 mPaused = true; 481} 482 483 484void NuPlayer::Decoder::onFlush() { 485 doFlush(true); 486 487 if (isDiscontinuityPending()) { 488 // This could happen if the client starts seeking/shutdown 489 // after we queued an EOS for discontinuities. 490 // We can consider discontinuity handled. 491 finishHandleDiscontinuity(false /* flushOnTimeChange */); 492 } 493 494 sp<AMessage> notify = mNotify->dup(); 495 notify->setInt32("what", kWhatFlushCompleted); 496 notify->post(); 497} 498 499void NuPlayer::Decoder::onShutdown(bool notifyComplete) { 500 status_t err = OK; 501 502 // if there is a pending resume request, notify complete now 503 notifyResumeCompleteIfNecessary(); 504 505 if (mCodec != NULL) { 506 err = mCodec->release(); 507 mCodec = NULL; 508 ++mBufferGeneration; 509 510 if (mSurface != NULL) { 511 // reconnect to surface as MediaCodec disconnected from it 512 status_t error = nativeWindowConnect(mSurface.get(), "onShutdown"); 513 ALOGW_IF(error != NO_ERROR, 514 "[%s] failed to connect to native window, error=%d", 515 mComponentName.c_str(), error); 516 } 517 mComponentName = "decoder"; 518 } 519 520 releaseAndResetMediaBuffers(); 521 522 if (err != OK) { 523 ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err); 524 handleError(err); 525 // finish with posting kWhatShutdownCompleted. 526 } 527 528 if (notifyComplete) { 529 sp<AMessage> notify = mNotify->dup(); 530 notify->setInt32("what", kWhatShutdownCompleted); 531 notify->post(); 532 mPaused = true; 533 } 534} 535 536/* 537 * returns true if we should request more data 538 */ 539bool NuPlayer::Decoder::doRequestBuffers() { 540 if (isDiscontinuityPending()) { 541 return false; 542 } 543 status_t err = OK; 544 while (err == OK && !mDequeuedInputBuffers.empty()) { 545 size_t bufferIx = *mDequeuedInputBuffers.begin(); 546 sp<AMessage> msg = new AMessage(); 547 msg->setSize("buffer-ix", bufferIx); 548 err = fetchInputData(msg); 549 if (err != OK && err != ERROR_END_OF_STREAM) { 550 // if EOS, need to queue EOS buffer 551 break; 552 } 553 mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin()); 554 555 if (!mPendingInputMessages.empty() 556 || !onInputBufferFetched(msg)) { 557 mPendingInputMessages.push_back(msg); 558 } 559 } 560 561 return err == -EWOULDBLOCK 562 && mSource->feedMoreTSData() == OK; 563} 564 565void NuPlayer::Decoder::handleError(int32_t err) 566{ 567 // We cannot immediately release the codec due to buffers still outstanding 568 // in the renderer. We signal to the player the error so it can shutdown/release the 569 // decoder after flushing and increment the generation to discard unnecessary messages. 570 571 ++mBufferGeneration; 572 573 sp<AMessage> notify = mNotify->dup(); 574 notify->setInt32("what", kWhatError); 575 notify->setInt32("err", err); 576 notify->post(); 577} 578 579status_t NuPlayer::Decoder::releaseCrypto() 580{ 581 ALOGV("releaseCrypto"); 582 583 sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this); 584 585 sp<AMessage> response; 586 status_t status = msg->postAndAwaitResponse(&response); 587 if (status == OK && response != NULL) { 588 CHECK(response->findInt32("status", &status)); 589 ALOGV("releaseCrypto ret: %d ", status); 590 } else { 591 ALOGE("releaseCrypto err: %d", status); 592 } 593 594 return status; 595} 596 597void NuPlayer::Decoder::onReleaseCrypto(const sp<AMessage>& msg) 598{ 599 status_t status = INVALID_OPERATION; 600 if (mCodec != NULL) { 601 status = mCodec->releaseCrypto(); 602 } else { 603 // returning OK if the codec has been already released 604 status = OK; 605 ALOGE("onReleaseCrypto No mCodec. err: %d", status); 606 } 607 608 sp<AMessage> response = new AMessage; 609 response->setInt32("status", status); 610 611 sp<AReplyToken> replyID; 612 CHECK(msg->senderAwaitsResponse(&replyID)); 613 response->postReply(replyID); 614} 615 616bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) { 617 if (isDiscontinuityPending()) { 618 return false; 619 } 620 621 sp<MediaCodecBuffer> buffer; 622 mCodec->getInputBuffer(index, &buffer); 623 624 if (buffer == NULL) { 625 handleError(UNKNOWN_ERROR); 626 return false; 627 } 628 629 if (index >= mInputBuffers.size()) { 630 for (size_t i = mInputBuffers.size(); i <= index; ++i) { 631 mInputBuffers.add(); 632 mMediaBuffers.add(); 633 mInputBufferIsDequeued.add(); 634 mMediaBuffers.editItemAt(i) = NULL; 635 mInputBufferIsDequeued.editItemAt(i) = false; 636 } 637 } 638 mInputBuffers.editItemAt(index) = buffer; 639 640 //CHECK_LT(bufferIx, mInputBuffers.size()); 641 642 if (mMediaBuffers[index] != NULL) { 643 mMediaBuffers[index]->release(); 644 mMediaBuffers.editItemAt(index) = NULL; 645 } 646 mInputBufferIsDequeued.editItemAt(index) = true; 647 648 if (!mCSDsToSubmit.isEmpty()) { 649 sp<AMessage> msg = new AMessage(); 650 msg->setSize("buffer-ix", index); 651 652 sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0); 653 ALOGI("[%s] resubmitting CSD", mComponentName.c_str()); 654 msg->setBuffer("buffer", buffer); 655 mCSDsToSubmit.removeAt(0); 656 if (!onInputBufferFetched(msg)) { 657 handleError(UNKNOWN_ERROR); 658 return false; 659 } 660 return true; 661 } 662 663 while (!mPendingInputMessages.empty()) { 664 sp<AMessage> msg = *mPendingInputMessages.begin(); 665 if (!onInputBufferFetched(msg)) { 666 break; 667 } 668 mPendingInputMessages.erase(mPendingInputMessages.begin()); 669 } 670 671 if (!mInputBufferIsDequeued.editItemAt(index)) { 672 return true; 673 } 674 675 mDequeuedInputBuffers.push_back(index); 676 677 onRequestInputBuffers(); 678 return true; 679} 680 681bool NuPlayer::Decoder::handleAnOutputBuffer( 682 size_t index, 683 size_t offset, 684 size_t size, 685 int64_t timeUs, 686 int32_t flags) { 687// CHECK_LT(bufferIx, mOutputBuffers.size()); 688 sp<MediaCodecBuffer> buffer; 689 mCodec->getOutputBuffer(index, &buffer); 690 691 if (buffer == NULL) { 692 handleError(UNKNOWN_ERROR); 693 return false; 694 } 695 696 if (index >= mOutputBuffers.size()) { 697 for (size_t i = mOutputBuffers.size(); i <= index; ++i) { 698 mOutputBuffers.add(); 699 } 700 } 701 702 mOutputBuffers.editItemAt(index) = buffer; 703 704 buffer->setRange(offset, size); 705 buffer->meta()->clear(); 706 buffer->meta()->setInt64("timeUs", timeUs); 707 708 bool eos = flags & MediaCodec::BUFFER_FLAG_EOS; 709 // we do not expect CODECCONFIG or SYNCFRAME for decoder 710 711 sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this); 712 reply->setSize("buffer-ix", index); 713 reply->setInt32("generation", mBufferGeneration); 714 715 if (eos) { 716 ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video"); 717 718 buffer->meta()->setInt32("eos", true); 719 reply->setInt32("eos", true); 720 } else if (mSkipRenderingUntilMediaTimeUs >= 0) { 721 if (timeUs < mSkipRenderingUntilMediaTimeUs) { 722 ALOGV("[%s] dropping buffer at time %lld as requested.", 723 mComponentName.c_str(), (long long)timeUs); 724 725 reply->post(); 726 return true; 727 } 728 729 mSkipRenderingUntilMediaTimeUs = -1; 730 } 731 732 mNumFramesTotal += !mIsAudio; 733 734 // wait until 1st frame comes out to signal resume complete 735 notifyResumeCompleteIfNecessary(); 736 737 if (mRenderer != NULL) { 738 // send the buffer to renderer. 739 mRenderer->queueBuffer(mIsAudio, buffer, reply); 740 if (eos && !isDiscontinuityPending()) { 741 mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM); 742 } 743 } 744 745 return true; 746} 747 748void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) { 749 if (!mIsAudio) { 750 int32_t width, height; 751 if (format->findInt32("width", &width) 752 && format->findInt32("height", &height)) { 753 mStats->setInt32("width", width); 754 mStats->setInt32("height", height); 755 } 756 sp<AMessage> notify = mNotify->dup(); 757 notify->setInt32("what", kWhatVideoSizeChanged); 758 notify->setMessage("format", format); 759 notify->post(); 760 } else if (mRenderer != NULL) { 761 uint32_t flags; 762 int64_t durationUs; 763 bool hasVideo = (mSource->getFormat(false /* audio */) != NULL); 764 if (getAudioDeepBufferSetting() // override regardless of source duration 765 || (!hasVideo 766 && mSource->getDuration(&durationUs) == OK 767 && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) { 768 flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER; 769 } else { 770 flags = AUDIO_OUTPUT_FLAG_NONE; 771 } 772 773 sp<AMessage> reply = new AMessage(kWhatAudioOutputFormatChanged, this); 774 reply->setInt32("generation", mBufferGeneration); 775 mRenderer->changeAudioFormat( 776 format, false /* offloadOnly */, hasVideo, 777 flags, mSource->isStreaming(), reply); 778 } 779} 780 781void NuPlayer::Decoder::releaseAndResetMediaBuffers() { 782 for (size_t i = 0; i < mMediaBuffers.size(); i++) { 783 if (mMediaBuffers[i] != NULL) { 784 mMediaBuffers[i]->release(); 785 mMediaBuffers.editItemAt(i) = NULL; 786 } 787 } 788 mMediaBuffers.resize(mInputBuffers.size()); 789 for (size_t i = 0; i < mMediaBuffers.size(); i++) { 790 mMediaBuffers.editItemAt(i) = NULL; 791 } 792 mInputBufferIsDequeued.clear(); 793 mInputBufferIsDequeued.resize(mInputBuffers.size()); 794 for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) { 795 mInputBufferIsDequeued.editItemAt(i) = false; 796 } 797 798 mPendingInputMessages.clear(); 799 mDequeuedInputBuffers.clear(); 800 mSkipRenderingUntilMediaTimeUs = -1; 801} 802 803void NuPlayer::Decoder::requestCodecNotification() { 804 if (mCodec != NULL) { 805 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this); 806 reply->setInt32("generation", mBufferGeneration); 807 mCodec->requestActivityNotification(reply); 808 } 809} 810 811bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) { 812 int32_t generation; 813 CHECK(msg->findInt32("generation", &generation)); 814 return generation != mBufferGeneration; 815} 816 817status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) { 818 sp<ABuffer> accessUnit; 819 bool dropAccessUnit = true; 820 do { 821 status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit); 822 823 if (err == -EWOULDBLOCK) { 824 return err; 825 } else if (err != OK) { 826 if (err == INFO_DISCONTINUITY) { 827 int32_t type; 828 CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); 829 830 bool formatChange = 831 (mIsAudio && 832 (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT)) 833 || (!mIsAudio && 834 (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT)); 835 836 bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0; 837 838 ALOGI("%s discontinuity (format=%d, time=%d)", 839 mIsAudio ? "audio" : "video", formatChange, timeChange); 840 841 bool seamlessFormatChange = false; 842 sp<AMessage> newFormat = mSource->getFormat(mIsAudio); 843 if (formatChange) { 844 seamlessFormatChange = 845 supportsSeamlessFormatChange(newFormat); 846 // treat seamless format change separately 847 formatChange = !seamlessFormatChange; 848 } 849 850 // For format or time change, return EOS to queue EOS input, 851 // then wait for EOS on output. 852 if (formatChange /* not seamless */) { 853 mFormatChangePending = true; 854 err = ERROR_END_OF_STREAM; 855 } else if (timeChange) { 856 rememberCodecSpecificData(newFormat); 857 mTimeChangePending = true; 858 err = ERROR_END_OF_STREAM; 859 } else if (seamlessFormatChange) { 860 // reuse existing decoder and don't flush 861 rememberCodecSpecificData(newFormat); 862 continue; 863 } else { 864 // This stream is unaffected by the discontinuity 865 return -EWOULDBLOCK; 866 } 867 } 868 869 // reply should only be returned without a buffer set 870 // when there is an error (including EOS) 871 CHECK(err != OK); 872 873 reply->setInt32("err", err); 874 return ERROR_END_OF_STREAM; 875 } 876 877 dropAccessUnit = false; 878 if (!mIsAudio && !mIsSecure) { 879 int32_t layerId = 0; 880 bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId); 881 if (mRenderer->getVideoLateByUs() > 100000ll 882 && mIsVideoAVC 883 && !IsAVCReferenceFrame(accessUnit)) { 884 dropAccessUnit = true; 885 } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) { 886 // Add only one layer each time. 887 if (layerId > mCurrentMaxVideoTemporalLayerId + 1 888 || layerId >= mNumVideoTemporalLayerAllowed) { 889 dropAccessUnit = true; 890 ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d", 891 layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed, 892 mCurrentMaxVideoTemporalLayerId); 893 } else if (layerId > mCurrentMaxVideoTemporalLayerId) { 894 mCurrentMaxVideoTemporalLayerId = layerId; 895 } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) { 896 mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1; 897 } 898 } 899 if (dropAccessUnit) { 900 if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) { 901 mCurrentMaxVideoTemporalLayerId = layerId - 1; 902 } 903 ++mNumInputFramesDropped; 904 } 905 } 906 } while (dropAccessUnit); 907 908 // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video"); 909#if 0 910 int64_t mediaTimeUs; 911 CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); 912 ALOGV("[%s] feeding input buffer at media time %.3f", 913 mIsAudio ? "audio" : "video", 914 mediaTimeUs / 1E6); 915#endif 916 917 if (mCCDecoder != NULL) { 918 mCCDecoder->decode(accessUnit); 919 } 920 921 reply->setBuffer("buffer", accessUnit); 922 923 return OK; 924} 925 926bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) { 927 size_t bufferIx; 928 CHECK(msg->findSize("buffer-ix", &bufferIx)); 929 CHECK_LT(bufferIx, mInputBuffers.size()); 930 sp<MediaCodecBuffer> codecBuffer = mInputBuffers[bufferIx]; 931 932 sp<ABuffer> buffer; 933 bool hasBuffer = msg->findBuffer("buffer", &buffer); 934 bool needsCopy = true; 935 936 if (buffer == NULL /* includes !hasBuffer */) { 937 int32_t streamErr = ERROR_END_OF_STREAM; 938 CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); 939 940 CHECK(streamErr != OK); 941 942 // attempt to queue EOS 943 status_t err = mCodec->queueInputBuffer( 944 bufferIx, 945 0, 946 0, 947 0, 948 MediaCodec::BUFFER_FLAG_EOS); 949 if (err == OK) { 950 mInputBufferIsDequeued.editItemAt(bufferIx) = false; 951 } else if (streamErr == ERROR_END_OF_STREAM) { 952 streamErr = err; 953 // err will not be ERROR_END_OF_STREAM 954 } 955 956 if (streamErr != ERROR_END_OF_STREAM) { 957 ALOGE("Stream error for %s (err=%d), EOS %s queued", 958 mComponentName.c_str(), 959 streamErr, 960 err == OK ? "successfully" : "unsuccessfully"); 961 handleError(streamErr); 962 } 963 } else { 964 sp<AMessage> extra; 965 if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) { 966 int64_t resumeAtMediaTimeUs; 967 if (extra->findInt64( 968 "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) { 969 ALOGI("[%s] suppressing rendering until %lld us", 970 mComponentName.c_str(), (long long)resumeAtMediaTimeUs); 971 mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs; 972 } 973 } 974 975 int64_t timeUs = 0; 976 uint32_t flags = 0; 977 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 978 979 int32_t eos, csd; 980 // we do not expect SYNCFRAME for decoder 981 if (buffer->meta()->findInt32("eos", &eos) && eos) { 982 flags |= MediaCodec::BUFFER_FLAG_EOS; 983 } else if (buffer->meta()->findInt32("csd", &csd) && csd) { 984 flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG; 985 } 986 987 // Modular DRM 988 MediaBuffer *mediaBuf = NULL; 989 NuPlayerDrm::CryptoInfo *cryptInfo = NULL; 990 991 // copy into codec buffer 992 if (needsCopy) { 993 if (buffer->size() > codecBuffer->capacity()) { 994 handleError(ERROR_BUFFER_TOO_SMALL); 995 mDequeuedInputBuffers.push_back(bufferIx); 996 return false; 997 } 998 999 if (buffer->data() != NULL) { 1000 codecBuffer->setRange(0, buffer->size()); 1001 memcpy(codecBuffer->data(), buffer->data(), buffer->size()); 1002 } else { // No buffer->data() 1003 //Modular DRM 1004 mediaBuf = (MediaBuffer*)buffer->getMediaBufferBase(); 1005 if (mediaBuf != NULL) { 1006 codecBuffer->setRange(0, mediaBuf->size()); 1007 memcpy(codecBuffer->data(), mediaBuf->data(), mediaBuf->size()); 1008 1009 sp<MetaData> meta_data = mediaBuf->meta_data(); 1010 cryptInfo = NuPlayerDrm::getSampleCryptoInfo(meta_data); 1011 1012 // since getMediaBuffer() has incremented the refCount 1013 mediaBuf->release(); 1014 } else { // No mediaBuf 1015 ALOGE("onInputBufferFetched: buffer->data()/mediaBuf are NULL for %p", 1016 buffer.get()); 1017 handleError(UNKNOWN_ERROR); 1018 return false; 1019 } 1020 } // buffer->data() 1021 } // needsCopy 1022 1023 status_t err; 1024 AString errorDetailMsg; 1025 if (cryptInfo != NULL) { 1026 err = mCodec->queueSecureInputBuffer( 1027 bufferIx, 1028 codecBuffer->offset(), 1029 cryptInfo->subSamples, 1030 cryptInfo->numSubSamples, 1031 cryptInfo->key, 1032 cryptInfo->iv, 1033 cryptInfo->mode, 1034 cryptInfo->pattern, 1035 timeUs, 1036 flags, 1037 &errorDetailMsg); 1038 // synchronous call so done with cryptInfo here 1039 free(cryptInfo); 1040 } else { 1041 err = mCodec->queueInputBuffer( 1042 bufferIx, 1043 codecBuffer->offset(), 1044 codecBuffer->size(), 1045 timeUs, 1046 flags, 1047 &errorDetailMsg); 1048 } // no cryptInfo 1049 1050 if (err != OK) { 1051 ALOGE("onInputBufferFetched: queue%sInputBuffer failed for %s (err=%d, %s)", 1052 (cryptInfo != NULL ? "Secure" : ""), 1053 mComponentName.c_str(), err, errorDetailMsg.c_str()); 1054 handleError(err); 1055 } else { 1056 mInputBufferIsDequeued.editItemAt(bufferIx) = false; 1057 } 1058 1059 } // buffer != NULL 1060 return true; 1061} 1062 1063void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) { 1064 status_t err; 1065 int32_t render; 1066 size_t bufferIx; 1067 int32_t eos; 1068 CHECK(msg->findSize("buffer-ix", &bufferIx)); 1069 1070 if (!mIsAudio) { 1071 int64_t timeUs; 1072 sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx]; 1073 buffer->meta()->findInt64("timeUs", &timeUs); 1074 1075 if (mCCDecoder != NULL && mCCDecoder->isSelected()) { 1076 mCCDecoder->display(timeUs); 1077 } 1078 } 1079 1080 if (msg->findInt32("render", &render) && render) { 1081 int64_t timestampNs; 1082 CHECK(msg->findInt64("timestampNs", ×tampNs)); 1083 err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs); 1084 } else { 1085 mNumOutputFramesDropped += !mIsAudio; 1086 err = mCodec->releaseOutputBuffer(bufferIx); 1087 } 1088 if (err != OK) { 1089 ALOGE("failed to release output buffer for %s (err=%d)", 1090 mComponentName.c_str(), err); 1091 handleError(err); 1092 } 1093 if (msg->findInt32("eos", &eos) && eos 1094 && isDiscontinuityPending()) { 1095 finishHandleDiscontinuity(true /* flushOnTimeChange */); 1096 } 1097} 1098 1099bool NuPlayer::Decoder::isDiscontinuityPending() const { 1100 return mFormatChangePending || mTimeChangePending; 1101} 1102 1103void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) { 1104 ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d", 1105 mFormatChangePending, mTimeChangePending, flushOnTimeChange); 1106 1107 // If we have format change, pause and wait to be killed; 1108 // If we have time change only, flush and restart fetching. 1109 1110 if (mFormatChangePending) { 1111 mPaused = true; 1112 } else if (mTimeChangePending) { 1113 if (flushOnTimeChange) { 1114 doFlush(false /* notifyComplete */); 1115 signalResume(false /* notifyComplete */); 1116 } 1117 } 1118 1119 // Notify NuPlayer to either shutdown decoder, or rescan sources 1120 sp<AMessage> msg = mNotify->dup(); 1121 msg->setInt32("what", kWhatInputDiscontinuity); 1122 msg->setInt32("formatChange", mFormatChangePending); 1123 msg->post(); 1124 1125 mFormatChangePending = false; 1126 mTimeChangePending = false; 1127} 1128 1129bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange( 1130 const sp<AMessage> &targetFormat) const { 1131 if (targetFormat == NULL) { 1132 return true; 1133 } 1134 1135 AString mime; 1136 if (!targetFormat->findString("mime", &mime)) { 1137 return false; 1138 } 1139 1140 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) { 1141 // field-by-field comparison 1142 const char * keys[] = { "channel-count", "sample-rate", "is-adts" }; 1143 for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) { 1144 int32_t oldVal, newVal; 1145 if (!mInputFormat->findInt32(keys[i], &oldVal) || 1146 !targetFormat->findInt32(keys[i], &newVal) || 1147 oldVal != newVal) { 1148 return false; 1149 } 1150 } 1151 1152 sp<ABuffer> oldBuf, newBuf; 1153 if (mInputFormat->findBuffer("csd-0", &oldBuf) && 1154 targetFormat->findBuffer("csd-0", &newBuf)) { 1155 if (oldBuf->size() != newBuf->size()) { 1156 return false; 1157 } 1158 return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size()); 1159 } 1160 } 1161 return false; 1162} 1163 1164bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const { 1165 if (mInputFormat == NULL) { 1166 return false; 1167 } 1168 1169 if (targetFormat == NULL) { 1170 return true; 1171 } 1172 1173 AString oldMime, newMime; 1174 if (!mInputFormat->findString("mime", &oldMime) 1175 || !targetFormat->findString("mime", &newMime) 1176 || !(oldMime == newMime)) { 1177 return false; 1178 } 1179 1180 bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/")); 1181 bool seamless; 1182 if (audio) { 1183 seamless = supportsSeamlessAudioFormatChange(targetFormat); 1184 } else { 1185 int32_t isAdaptive; 1186 seamless = (mCodec != NULL && 1187 mInputFormat->findInt32("adaptive-playback", &isAdaptive) && 1188 isAdaptive); 1189 } 1190 1191 ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str()); 1192 return seamless; 1193} 1194 1195void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) { 1196 if (format == NULL) { 1197 return; 1198 } 1199 mCSDsForCurrentFormat.clear(); 1200 for (int32_t i = 0; ; ++i) { 1201 AString tag = "csd-"; 1202 tag.append(i); 1203 sp<ABuffer> buffer; 1204 if (!format->findBuffer(tag.c_str(), &buffer)) { 1205 break; 1206 } 1207 mCSDsForCurrentFormat.push(buffer); 1208 } 1209} 1210 1211void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() { 1212 if (mResumePending) { 1213 mResumePending = false; 1214 1215 sp<AMessage> notify = mNotify->dup(); 1216 notify->setInt32("what", kWhatResumeCompleted); 1217 notify->post(); 1218 } 1219} 1220 1221} // namespace android 1222 1223