NuPlayerDecoder.cpp revision 9fb815244cc33927e14ce0fe70539786a611daf9 
1/*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerDecoder"
19#include <utils/Log.h>
20#include <inttypes.h>
21
22#include <algorithm>
23
24#include "NuPlayerCCDecoder.h"
25#include "NuPlayerDecoder.h"
26#include "NuPlayerRenderer.h"
27#include "NuPlayerSource.h"
28
29#include <cutils/properties.h>
30#include <media/ICrypto.h>
31#include <media/stagefright/foundation/ABuffer.h>
32#include <media/stagefright/foundation/ADebug.h>
33#include <media/stagefright/foundation/AMessage.h>
34#include <media/stagefright/MediaBuffer.h>
35#include <media/stagefright/MediaCodec.h>
36#include <media/stagefright/MediaDefs.h>
37#include <media/stagefright/MediaErrors.h>
38
39#include <gui/Surface.h>
40
41#include "avc_utils.h"
42#include "ATSParser.h"
43
44namespace android {
45
46static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display
47
48// The default total video frame rate of a stream when that info is not available from
49// the source.
50static float kDefaultVideoFrameRateTotal = 30.f;
51
52static inline bool getAudioDeepBufferSetting() {
53    return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
54}
55
56NuPlayer::Decoder::Decoder(
57        const sp<AMessage> &notify,
58        const sp<Source> &source,
59        pid_t pid,
60        const sp<Renderer> &renderer,
61        const sp<Surface> &surface,
62        const sp<CCDecoder> &ccDecoder)
63    : DecoderBase(notify),
64      mSurface(surface),
65      mSource(source),
66      mRenderer(renderer),
67      mCCDecoder(ccDecoder),
68      mPid(pid),
69      mSkipRenderingUntilMediaTimeUs(-1ll),
70      mNumFramesTotal(0ll),
71      mNumInputFramesDropped(0ll),
72      mNumOutputFramesDropped(0ll),
73      mVideoWidth(0),
74      mVideoHeight(0),
75      mIsAudio(true),
76      mIsVideoAVC(false),
77      mIsSecure(false),
78      mFormatChangePending(false),
79      mTimeChangePending(false),
80      mFrameRateTotal(kDefaultVideoFrameRateTotal),
81      mPlaybackSpeed(1.0f),
82      mNumVideoTemporalLayerTotal(1), // decode all layers
83      mNumVideoTemporalLayerAllowed(1),
84      mCurrentMaxVideoTemporalLayerId(0),
85      mResumePending(false),
86      mComponentName("decoder") {
87    mCodecLooper = new ALooper;
88    mCodecLooper->setName("NPDecoder-CL");
89    mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
90    mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;
91}
92
93NuPlayer::Decoder::~Decoder() {
94    mCodec->release();
95    releaseAndResetMediaBuffers();
96}
97
98sp<AMessage> NuPlayer::Decoder::getStats() const {
99    mStats->setInt64("frames-total", mNumFramesTotal);
100    mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
101    mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
102    return mStats;
103}
104
105status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
106    if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
107        return BAD_VALUE;
108    }
109
110    sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
111
112    msg->setObject("surface", surface);
113    sp<AMessage> response;
114    status_t err = msg->postAndAwaitResponse(&response);
115    if (err == OK && response != NULL) {
116        CHECK(response->findInt32("err", &err));
117    }
118    return err;
119}
120
121void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
122    ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
123
124    switch (msg->what()) {
125        case kWhatCodecNotify:
126        {
127            int32_t cbID;
128            CHECK(msg->findInt32("callbackID", &cbID));
129
130            ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
131                    mIsAudio ? "audio" : "video", cbID, mPaused);
132
133            if (mPaused) {
134                break;
135            }
136
137            switch (cbID) {
138                case MediaCodec::CB_INPUT_AVAILABLE:
139                {
140                    int32_t index;
141                    CHECK(msg->findInt32("index", &index));
142
143                    handleAnInputBuffer(index);
144                    break;
145                }
146
147                case MediaCodec::CB_OUTPUT_AVAILABLE:
148                {
149                    int32_t index;
150                    size_t offset;
151                    size_t size;
152                    int64_t timeUs;
153                    int32_t flags;
154
155                    CHECK(msg->findInt32("index", &index));
156                    CHECK(msg->findSize("offset", &offset));
157                    CHECK(msg->findSize("size", &size));
158                    CHECK(msg->findInt64("timeUs", &timeUs));
159                    CHECK(msg->findInt32("flags", &flags));
160
161                    handleAnOutputBuffer(index, offset, size, timeUs, flags);
162                    break;
163                }
164
165                case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
166                {
167                    sp<AMessage> format;
168                    CHECK(msg->findMessage("format", &format));
169
170                    handleOutputFormatChange(format);
171                    break;
172                }
173
174                case MediaCodec::CB_ERROR:
175                {
176                    status_t err;
177                    CHECK(msg->findInt32("err", &err));
178                    ALOGE("Decoder (%s) reported error : 0x%x",
179                            mIsAudio ? "audio" : "video", err);
180
181                    handleError(err);
182                    break;
183                }
184
185                default:
186                {
187                    TRESPASS();
188                    break;
189                }
190            }
191
192            break;
193        }
194
195        case kWhatRenderBuffer:
196        {
197            if (!isStaleReply(msg)) {
198                onRenderBuffer(msg);
199            }
200            break;
201        }
202
203        case kWhatSetVideoSurface:
204        {
205            sp<AReplyToken> replyID;
206            CHECK(msg->senderAwaitsResponse(&replyID));
207
208            sp<RefBase> obj;
209            CHECK(msg->findObject("surface", &obj));
210            sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
211            int32_t err = INVALID_OPERATION;
212            // NOTE: in practice mSurface is always non-null, but checking here for completeness
213            if (mCodec != NULL && mSurface != NULL) {
214                // TODO: once AwesomePlayer is removed, remove this automatic connecting
215                // to the surface by MediaPlayerService.
216                //
217                // at this point MediaPlayerService::client has already connected to the
218                // surface, which MediaCodec does not expect
219                err = native_window_api_disconnect(surface.get(), NATIVE_WINDOW_API_MEDIA);
220                if (err == OK) {
221                    err = mCodec->setSurface(surface);
222                    ALOGI_IF(err, "codec setSurface returned: %d", err);
223                    if (err == OK) {
224                        // reconnect to the old surface as MPS::Client will expect to
225                        // be able to disconnect from it.
226                        (void)native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
227                        mSurface = surface;
228                    }
229                }
230                if (err != OK) {
231                    // reconnect to the new surface on error as MPS::Client will expect to
232                    // be able to disconnect from it.
233                    (void)native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA);
234                }
235            }
236
237            sp<AMessage> response = new AMessage;
238            response->setInt32("err", err);
239            response->postReply(replyID);
240            break;
241        }
242
243        default:
244            DecoderBase::onMessageReceived(msg);
245            break;
246    }
247}
248
249void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
250    CHECK(mCodec == NULL);
251
252    mFormatChangePending = false;
253    mTimeChangePending = false;
254
255    ++mBufferGeneration;
256
257    AString mime;
258    CHECK(format->findString("mime", &mime));
259
260    mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
261    mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
262
263    mComponentName = mime;
264    mComponentName.append(" decoder");
265    ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
266
267    mCodec = MediaCodec::CreateByType(
268            mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid);
269    int32_t secure = 0;
270    if (format->findInt32("secure", &secure) && secure != 0) {
271        if (mCodec != NULL) {
272            mCodec->getName(&mComponentName);
273            mComponentName.append(".secure");
274            mCodec->release();
275            ALOGI("[%s] creating", mComponentName.c_str());
276            mCodec = MediaCodec::CreateByComponentName(
277                    mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid);
278        }
279    }
280    if (mCodec == NULL) {
281        ALOGE("Failed to create %s%s decoder",
282                (secure ? "secure " : ""), mime.c_str());
283        handleError(UNKNOWN_ERROR);
284        return;
285    }
286    mIsSecure = secure;
287
288    mCodec->getName(&mComponentName);
289
290    status_t err;
291    if (mSurface != NULL) {
292        // disconnect from surface as MediaCodec will reconnect
293        err = native_window_api_disconnect(
294                mSurface.get(), NATIVE_WINDOW_API_MEDIA);
295        // We treat this as a warning, as this is a preparatory step.
296        // Codec will try to connect to the surface, which is where
297        // any error signaling will occur.
298        ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
299    }
300    err = mCodec->configure(
301            format, mSurface, NULL /* crypto */, 0 /* flags */);
302    if (err != OK) {
303        ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
304        mCodec->release();
305        mCodec.clear();
306        handleError(err);
307        return;
308    }
309    rememberCodecSpecificData(format);
310
311    // the following should work in configured state
312    CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
313    CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
314
315    mStats->setString("mime", mime.c_str());
316    mStats->setString("component-name", mComponentName.c_str());
317
318    if (!mIsAudio) {
319        int32_t width, height;
320        if (mOutputFormat->findInt32("width", &width)
321                && mOutputFormat->findInt32("height", &height)) {
322            mStats->setInt32("width", width);
323            mStats->setInt32("height", height);
324        }
325    }
326
327    sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
328    mCodec->setCallback(reply);
329
330    err = mCodec->start();
331    if (err != OK) {
332        ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
333        mCodec->release();
334        mCodec.clear();
335        handleError(err);
336        return;
337    }
338
339    releaseAndResetMediaBuffers();
340
341    mPaused = false;
342    mResumePending = false;
343}
344
345void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) {
346    bool needAdjustLayers = false;
347    float frameRateTotal;
348    if (params->findFloat("frame-rate-total", &frameRateTotal)
349            && mFrameRateTotal != frameRateTotal) {
350        needAdjustLayers = true;
351        mFrameRateTotal = frameRateTotal;
352    }
353
354    int32_t numVideoTemporalLayerTotal;
355    if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal)
356            && numVideoTemporalLayerTotal >= 0
357            && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers
358            && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) {
359        needAdjustLayers = true;
360        mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1);
361    }
362
363    if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) {
364        // TODO: For now, layer fps is calculated for some specific architectures.
365        // But it really should be extracted from the stream.
366        mVideoTemporalLayerAggregateFps[0] =
367            mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1));
368        for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) {
369            mVideoTemporalLayerAggregateFps[i] =
370                mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i))
371                + mVideoTemporalLayerAggregateFps[i - 1];
372        }
373    }
374
375    float playbackSpeed;
376    if (params->findFloat("playback-speed", &playbackSpeed)
377            && mPlaybackSpeed != playbackSpeed) {
378        needAdjustLayers = true;
379        mPlaybackSpeed = playbackSpeed;
380    }
381
382    if (needAdjustLayers) {
383        float decodeFrameRate = mFrameRateTotal;
384        // enable temporal layering optimization only if we know the layering depth
385        if (mNumVideoTemporalLayerTotal > 1) {
386            int32_t layerId;
387            for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) {
388                if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed
389                        >= kDisplayRefreshingRate * 0.9) {
390                    break;
391                }
392            }
393            mNumVideoTemporalLayerAllowed = layerId + 1;
394            decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId];
395        }
396        ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g",
397                mNumVideoTemporalLayerAllowed, decodeFrameRate);
398
399        if (mCodec == NULL) {
400            ALOGW("onSetParameters called before codec is created.");
401            return;
402        }
403
404        sp<AMessage> codecParams = new AMessage();
405        codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed);
406        mCodec->setParameters(codecParams);
407    }
408}
409
410void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
411    bool hadNoRenderer = (mRenderer == NULL);
412    mRenderer = renderer;
413    if (hadNoRenderer && mRenderer != NULL) {
414        // this means that the widevine legacy source is ready
415        onRequestInputBuffers();
416    }
417}
418
419void NuPlayer::Decoder::onGetInputBuffers(
420        Vector<sp<ABuffer> > *dstBuffers) {
421    CHECK_EQ((status_t)OK, mCodec->getWidevineLegacyBuffers(dstBuffers));
422}
423
424void NuPlayer::Decoder::onResume(bool notifyComplete) {
425    mPaused = false;
426
427    if (notifyComplete) {
428        mResumePending = true;
429    }
430    mCodec->start();
431}
432
433void NuPlayer::Decoder::doFlush(bool notifyComplete) {
434    if (mCCDecoder != NULL) {
435        mCCDecoder->flush();
436    }
437
438    if (mRenderer != NULL) {
439        mRenderer->flush(mIsAudio, notifyComplete);
440        mRenderer->signalTimeDiscontinuity();
441    }
442
443    status_t err = OK;
444    if (mCodec != NULL) {
445        err = mCodec->flush();
446        mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
447        ++mBufferGeneration;
448    }
449
450    if (err != OK) {
451        ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
452        handleError(err);
453        // finish with posting kWhatFlushCompleted.
454        // we attempt to release the buffers even if flush fails.
455    }
456    releaseAndResetMediaBuffers();
457    mPaused = true;
458}
459
460
461void NuPlayer::Decoder::onFlush() {
462    doFlush(true);
463
464    if (isDiscontinuityPending()) {
465        // This could happen if the client starts seeking/shutdown
466        // after we queued an EOS for discontinuities.
467        // We can consider discontinuity handled.
468        finishHandleDiscontinuity(false /* flushOnTimeChange */);
469    }
470
471    sp<AMessage> notify = mNotify->dup();
472    notify->setInt32("what", kWhatFlushCompleted);
473    notify->post();
474}
475
476void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
477    status_t err = OK;
478
479    // if there is a pending resume request, notify complete now
480    notifyResumeCompleteIfNecessary();
481
482    if (mCodec != NULL) {
483        err = mCodec->release();
484        mCodec = NULL;
485        ++mBufferGeneration;
486
487        if (mSurface != NULL) {
488            // reconnect to surface as MediaCodec disconnected from it
489            status_t error =
490                    native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
491            ALOGW_IF(error != NO_ERROR,
492                    "[%s] failed to connect to native window, error=%d",
493                    mComponentName.c_str(), error);
494        }
495        mComponentName = "decoder";
496    }
497
498    releaseAndResetMediaBuffers();
499
500    if (err != OK) {
501        ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
502        handleError(err);
503        // finish with posting kWhatShutdownCompleted.
504    }
505
506    if (notifyComplete) {
507        sp<AMessage> notify = mNotify->dup();
508        notify->setInt32("what", kWhatShutdownCompleted);
509        notify->post();
510        mPaused = true;
511    }
512}
513
514/*
515 * returns true if we should request more data
516 */
517bool NuPlayer::Decoder::doRequestBuffers() {
518    // mRenderer is only NULL if we have a legacy widevine source that
519    // is not yet ready. In this case we must not fetch input.
520    if (isDiscontinuityPending() || mRenderer == NULL) {
521        return false;
522    }
523    status_t err = OK;
524    while (err == OK && !mDequeuedInputBuffers.empty()) {
525        size_t bufferIx = *mDequeuedInputBuffers.begin();
526        sp<AMessage> msg = new AMessage();
527        msg->setSize("buffer-ix", bufferIx);
528        err = fetchInputData(msg);
529        if (err != OK && err != ERROR_END_OF_STREAM) {
530            // if EOS, need to queue EOS buffer
531            break;
532        }
533        mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
534
535        if (!mPendingInputMessages.empty()
536                || !onInputBufferFetched(msg)) {
537            mPendingInputMessages.push_back(msg);
538        }
539    }
540
541    return err == -EWOULDBLOCK
542            && mSource->feedMoreTSData() == OK;
543}
544
545void NuPlayer::Decoder::handleError(int32_t err)
546{
547    // We cannot immediately release the codec due to buffers still outstanding
548    // in the renderer.  We signal to the player the error so it can shutdown/release the
549    // decoder after flushing and increment the generation to discard unnecessary messages.
550
551    ++mBufferGeneration;
552
553    sp<AMessage> notify = mNotify->dup();
554    notify->setInt32("what", kWhatError);
555    notify->setInt32("err", err);
556    notify->post();
557}
558
559bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
560    if (isDiscontinuityPending()) {
561        return false;
562    }
563
564    sp<ABuffer> buffer;
565    mCodec->getInputBuffer(index, &buffer);
566
567    if (buffer == NULL) {
568        handleError(UNKNOWN_ERROR);
569        return false;
570    }
571
572    if (index >= mInputBuffers.size()) {
573        for (size_t i = mInputBuffers.size(); i <= index; ++i) {
574            mInputBuffers.add();
575            mMediaBuffers.add();
576            mInputBufferIsDequeued.add();
577            mMediaBuffers.editItemAt(i) = NULL;
578            mInputBufferIsDequeued.editItemAt(i) = false;
579        }
580    }
581    mInputBuffers.editItemAt(index) = buffer;
582
583    //CHECK_LT(bufferIx, mInputBuffers.size());
584
585    if (mMediaBuffers[index] != NULL) {
586        mMediaBuffers[index]->release();
587        mMediaBuffers.editItemAt(index) = NULL;
588    }
589    mInputBufferIsDequeued.editItemAt(index) = true;
590
591    if (!mCSDsToSubmit.isEmpty()) {
592        sp<AMessage> msg = new AMessage();
593        msg->setSize("buffer-ix", index);
594
595        sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
596        ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
597        msg->setBuffer("buffer", buffer);
598        mCSDsToSubmit.removeAt(0);
599        if (!onInputBufferFetched(msg)) {
600            handleError(UNKNOWN_ERROR);
601            return false;
602        }
603        return true;
604    }
605
606    while (!mPendingInputMessages.empty()) {
607        sp<AMessage> msg = *mPendingInputMessages.begin();
608        if (!onInputBufferFetched(msg)) {
609            break;
610        }
611        mPendingInputMessages.erase(mPendingInputMessages.begin());
612    }
613
614    if (!mInputBufferIsDequeued.editItemAt(index)) {
615        return true;
616    }
617
618    mDequeuedInputBuffers.push_back(index);
619
620    onRequestInputBuffers();
621    return true;
622}
623
624bool NuPlayer::Decoder::handleAnOutputBuffer(
625        size_t index,
626        size_t offset,
627        size_t size,
628        int64_t timeUs,
629        int32_t flags) {
630//    CHECK_LT(bufferIx, mOutputBuffers.size());
631    sp<ABuffer> buffer;
632    mCodec->getOutputBuffer(index, &buffer);
633
634    if (index >= mOutputBuffers.size()) {
635        for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
636            mOutputBuffers.add();
637        }
638    }
639
640    mOutputBuffers.editItemAt(index) = buffer;
641
642    buffer->setRange(offset, size);
643    buffer->meta()->clear();
644    buffer->meta()->setInt64("timeUs", timeUs);
645
646    bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
647    // we do not expect CODECCONFIG or SYNCFRAME for decoder
648
649    sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
650    reply->setSize("buffer-ix", index);
651    reply->setInt32("generation", mBufferGeneration);
652
653    if (eos) {
654        ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
655
656        buffer->meta()->setInt32("eos", true);
657        reply->setInt32("eos", true);
658    } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
659        if (timeUs < mSkipRenderingUntilMediaTimeUs) {
660            ALOGV("[%s] dropping buffer at time %lld as requested.",
661                     mComponentName.c_str(), (long long)timeUs);
662
663            reply->post();
664            return true;
665        }
666
667        mSkipRenderingUntilMediaTimeUs = -1;
668    }
669
670    mNumFramesTotal += !mIsAudio;
671
672    // wait until 1st frame comes out to signal resume complete
673    notifyResumeCompleteIfNecessary();
674
675    if (mRenderer != NULL) {
676        // send the buffer to renderer.
677        mRenderer->queueBuffer(mIsAudio, buffer, reply);
678        if (eos && !isDiscontinuityPending()) {
679            mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
680        }
681    }
682
683    return true;
684}
685
686void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
687    if (!mIsAudio) {
688        int32_t width, height;
689        if (format->findInt32("width", &width)
690                && format->findInt32("height", &height)) {
691            mStats->setInt32("width", width);
692            mStats->setInt32("height", height);
693        }
694        sp<AMessage> notify = mNotify->dup();
695        notify->setInt32("what", kWhatVideoSizeChanged);
696        notify->setMessage("format", format);
697        notify->post();
698    } else if (mRenderer != NULL) {
699        uint32_t flags;
700        int64_t durationUs;
701        bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
702        if (getAudioDeepBufferSetting() // override regardless of source duration
703                || (!hasVideo
704                        && mSource->getDuration(&durationUs) == OK
705                        && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
706            flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
707        } else {
708            flags = AUDIO_OUTPUT_FLAG_NONE;
709        }
710
711        status_t err = mRenderer->openAudioSink(
712                format, false /* offloadOnly */, hasVideo, flags, NULL /* isOffloaed */);
713        if (err != OK) {
714            handleError(err);
715        }
716    }
717}
718
719void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
720    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
721        if (mMediaBuffers[i] != NULL) {
722            mMediaBuffers[i]->release();
723            mMediaBuffers.editItemAt(i) = NULL;
724        }
725    }
726    mMediaBuffers.resize(mInputBuffers.size());
727    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
728        mMediaBuffers.editItemAt(i) = NULL;
729    }
730    mInputBufferIsDequeued.clear();
731    mInputBufferIsDequeued.resize(mInputBuffers.size());
732    for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
733        mInputBufferIsDequeued.editItemAt(i) = false;
734    }
735
736    mPendingInputMessages.clear();
737    mDequeuedInputBuffers.clear();
738    mSkipRenderingUntilMediaTimeUs = -1;
739}
740
741void NuPlayer::Decoder::requestCodecNotification() {
742    if (mCodec != NULL) {
743        sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
744        reply->setInt32("generation", mBufferGeneration);
745        mCodec->requestActivityNotification(reply);
746    }
747}
748
749bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
750    int32_t generation;
751    CHECK(msg->findInt32("generation", &generation));
752    return generation != mBufferGeneration;
753}
754
755status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
756    sp<ABuffer> accessUnit;
757    bool dropAccessUnit;
758    do {
759        status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
760
761        if (err == -EWOULDBLOCK) {
762            return err;
763        } else if (err != OK) {
764            if (err == INFO_DISCONTINUITY) {
765                int32_t type;
766                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
767
768                bool formatChange =
769                    (mIsAudio &&
770                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
771                    || (!mIsAudio &&
772                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
773
774                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
775
776                ALOGI("%s discontinuity (format=%d, time=%d)",
777                        mIsAudio ? "audio" : "video", formatChange, timeChange);
778
779                bool seamlessFormatChange = false;
780                sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
781                if (formatChange) {
782                    seamlessFormatChange =
783                        supportsSeamlessFormatChange(newFormat);
784                    // treat seamless format change separately
785                    formatChange = !seamlessFormatChange;
786                }
787
788                // For format or time change, return EOS to queue EOS input,
789                // then wait for EOS on output.
790                if (formatChange /* not seamless */) {
791                    mFormatChangePending = true;
792                    err = ERROR_END_OF_STREAM;
793                } else if (timeChange) {
794                    rememberCodecSpecificData(newFormat);
795                    mTimeChangePending = true;
796                    err = ERROR_END_OF_STREAM;
797                } else if (seamlessFormatChange) {
798                    // reuse existing decoder and don't flush
799                    rememberCodecSpecificData(newFormat);
800                    continue;
801                } else {
802                    // This stream is unaffected by the discontinuity
803                    return -EWOULDBLOCK;
804                }
805            }
806
807            // reply should only be returned without a buffer set
808            // when there is an error (including EOS)
809            CHECK(err != OK);
810
811            reply->setInt32("err", err);
812            return ERROR_END_OF_STREAM;
813        }
814
815        dropAccessUnit = false;
816        if (!mIsAudio && !mIsSecure) {
817            int32_t layerId = 0;
818            bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId);
819            if (mRenderer->getVideoLateByUs() > 100000ll
820                    && mIsVideoAVC
821                    && !IsAVCReferenceFrame(accessUnit)) {
822                dropAccessUnit = true;
823            } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) {
824                // Add only one layer each time.
825                if (layerId > mCurrentMaxVideoTemporalLayerId + 1
826                        || layerId >= mNumVideoTemporalLayerAllowed) {
827                    dropAccessUnit = true;
828                    ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d",
829                            layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed,
830                            mCurrentMaxVideoTemporalLayerId);
831                } else if (layerId > mCurrentMaxVideoTemporalLayerId) {
832                    mCurrentMaxVideoTemporalLayerId = layerId;
833                } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) {
834                    mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1;
835                }
836            }
837            if (dropAccessUnit) {
838                if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) {
839                    mCurrentMaxVideoTemporalLayerId = layerId - 1;
840                }
841                ++mNumInputFramesDropped;
842            }
843        }
844    } while (dropAccessUnit);
845
846    // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
847#if 0
848    int64_t mediaTimeUs;
849    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
850    ALOGV("[%s] feeding input buffer at media time %.3f",
851         mIsAudio ? "audio" : "video",
852         mediaTimeUs / 1E6);
853#endif
854
855    if (mCCDecoder != NULL) {
856        mCCDecoder->decode(accessUnit);
857    }
858
859    reply->setBuffer("buffer", accessUnit);
860
861    return OK;
862}
863
864bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
865    size_t bufferIx;
866    CHECK(msg->findSize("buffer-ix", &bufferIx));
867    CHECK_LT(bufferIx, mInputBuffers.size());
868    sp<ABuffer> codecBuffer = mInputBuffers[bufferIx];
869
870    sp<ABuffer> buffer;
871    bool hasBuffer = msg->findBuffer("buffer", &buffer);
872
873    // handle widevine classic source - that fills an arbitrary input buffer
874    MediaBuffer *mediaBuffer = NULL;
875    if (hasBuffer) {
876        mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase());
877        if (mediaBuffer != NULL) {
878            // likely filled another buffer than we requested: adjust buffer index
879            size_t ix;
880            for (ix = 0; ix < mInputBuffers.size(); ix++) {
881                const sp<ABuffer> &buf = mInputBuffers[ix];
882                if (buf->data() == mediaBuffer->data()) {
883                    // all input buffers are dequeued on start, hence the check
884                    if (!mInputBufferIsDequeued[ix]) {
885                        ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu",
886                                mComponentName.c_str(), ix, bufferIx);
887                        mediaBuffer->release();
888                        return false;
889                    }
890
891                    // TRICKY: need buffer for the metadata, so instead, set
892                    // codecBuffer to the same (though incorrect) buffer to
893                    // avoid a memcpy into the codecBuffer
894                    codecBuffer = buffer;
895                    codecBuffer->setRange(
896                            mediaBuffer->range_offset(),
897                            mediaBuffer->range_length());
898                    bufferIx = ix;
899                    break;
900                }
901            }
902            CHECK(ix < mInputBuffers.size());
903        }
904    }
905
906    if (buffer == NULL /* includes !hasBuffer */) {
907        int32_t streamErr = ERROR_END_OF_STREAM;
908        CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
909
910        CHECK(streamErr != OK);
911
912        // attempt to queue EOS
913        status_t err = mCodec->queueInputBuffer(
914                bufferIx,
915                0,
916                0,
917                0,
918                MediaCodec::BUFFER_FLAG_EOS);
919        if (err == OK) {
920            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
921        } else if (streamErr == ERROR_END_OF_STREAM) {
922            streamErr = err;
923            // err will not be ERROR_END_OF_STREAM
924        }
925
926        if (streamErr != ERROR_END_OF_STREAM) {
927            ALOGE("Stream error for %s (err=%d), EOS %s queued",
928                    mComponentName.c_str(),
929                    streamErr,
930                    err == OK ? "successfully" : "unsuccessfully");
931            handleError(streamErr);
932        }
933    } else {
934        sp<AMessage> extra;
935        if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
936            int64_t resumeAtMediaTimeUs;
937            if (extra->findInt64(
938                        "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
939                ALOGI("[%s] suppressing rendering until %lld us",
940                        mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
941                mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
942            }
943        }
944
945        int64_t timeUs = 0;
946        uint32_t flags = 0;
947        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
948
949        int32_t eos, csd;
950        // we do not expect SYNCFRAME for decoder
951        if (buffer->meta()->findInt32("eos", &eos) && eos) {
952            flags |= MediaCodec::BUFFER_FLAG_EOS;
953        } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
954            flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
955        }
956
957        // copy into codec buffer
958        if (buffer != codecBuffer) {
959            if (buffer->size() > codecBuffer->capacity()) {
960                handleError(ERROR_BUFFER_TOO_SMALL);
961                mDequeuedInputBuffers.push_back(bufferIx);
962                return false;
963            }
964            codecBuffer->setRange(0, buffer->size());
965            memcpy(codecBuffer->data(), buffer->data(), buffer->size());
966        }
967
968        status_t err = mCodec->queueInputBuffer(
969                        bufferIx,
970                        codecBuffer->offset(),
971                        codecBuffer->size(),
972                        timeUs,
973                        flags);
974        if (err != OK) {
975            if (mediaBuffer != NULL) {
976                mediaBuffer->release();
977            }
978            ALOGE("Failed to queue input buffer for %s (err=%d)",
979                    mComponentName.c_str(), err);
980            handleError(err);
981        } else {
982            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
983            if (mediaBuffer != NULL) {
984                CHECK(mMediaBuffers[bufferIx] == NULL);
985                mMediaBuffers.editItemAt(bufferIx) = mediaBuffer;
986            }
987        }
988    }
989    return true;
990}
991
992void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
993    status_t err;
994    int32_t render;
995    size_t bufferIx;
996    int32_t eos;
997    CHECK(msg->findSize("buffer-ix", &bufferIx));
998
999    if (!mIsAudio) {
1000        int64_t timeUs;
1001        sp<ABuffer> buffer = mOutputBuffers[bufferIx];
1002        buffer->meta()->findInt64("timeUs", &timeUs);
1003
1004        if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
1005            mCCDecoder->display(timeUs);
1006        }
1007    }
1008
1009    if (msg->findInt32("render", &render) && render) {
1010        int64_t timestampNs;
1011        CHECK(msg->findInt64("timestampNs", &timestampNs));
1012        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
1013    } else {
1014        mNumOutputFramesDropped += !mIsAudio;
1015        err = mCodec->releaseOutputBuffer(bufferIx);
1016    }
1017    if (err != OK) {
1018        ALOGE("failed to release output buffer for %s (err=%d)",
1019                mComponentName.c_str(), err);
1020        handleError(err);
1021    }
1022    if (msg->findInt32("eos", &eos) && eos
1023            && isDiscontinuityPending()) {
1024        finishHandleDiscontinuity(true /* flushOnTimeChange */);
1025    }
1026}
1027
1028bool NuPlayer::Decoder::isDiscontinuityPending() const {
1029    return mFormatChangePending || mTimeChangePending;
1030}
1031
1032void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
1033    ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
1034            mFormatChangePending, mTimeChangePending, flushOnTimeChange);
1035
1036    // If we have format change, pause and wait to be killed;
1037    // If we have time change only, flush and restart fetching.
1038
1039    if (mFormatChangePending) {
1040        mPaused = true;
1041    } else if (mTimeChangePending) {
1042        if (flushOnTimeChange) {
1043            doFlush(false /* notifyComplete */);
1044            signalResume(false /* notifyComplete */);
1045        }
1046    }
1047
1048    // Notify NuPlayer to either shutdown decoder, or rescan sources
1049    sp<AMessage> msg = mNotify->dup();
1050    msg->setInt32("what", kWhatInputDiscontinuity);
1051    msg->setInt32("formatChange", mFormatChangePending);
1052    msg->post();
1053
1054    mFormatChangePending = false;
1055    mTimeChangePending = false;
1056}
1057
1058bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
1059        const sp<AMessage> &targetFormat) const {
1060    if (targetFormat == NULL) {
1061        return true;
1062    }
1063
1064    AString mime;
1065    if (!targetFormat->findString("mime", &mime)) {
1066        return false;
1067    }
1068
1069    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
1070        // field-by-field comparison
1071        const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
1072        for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
1073            int32_t oldVal, newVal;
1074            if (!mInputFormat->findInt32(keys[i], &oldVal) ||
1075                    !targetFormat->findInt32(keys[i], &newVal) ||
1076                    oldVal != newVal) {
1077                return false;
1078            }
1079        }
1080
1081        sp<ABuffer> oldBuf, newBuf;
1082        if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
1083                targetFormat->findBuffer("csd-0", &newBuf)) {
1084            if (oldBuf->size() != newBuf->size()) {
1085                return false;
1086            }
1087            return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
1088        }
1089    }
1090    return false;
1091}
1092
1093bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
1094    if (mInputFormat == NULL) {
1095        return false;
1096    }
1097
1098    if (targetFormat == NULL) {
1099        return true;
1100    }
1101
1102    AString oldMime, newMime;
1103    if (!mInputFormat->findString("mime", &oldMime)
1104            || !targetFormat->findString("mime", &newMime)
1105            || !(oldMime == newMime)) {
1106        return false;
1107    }
1108
1109    bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
1110    bool seamless;
1111    if (audio) {
1112        seamless = supportsSeamlessAudioFormatChange(targetFormat);
1113    } else {
1114        int32_t isAdaptive;
1115        seamless = (mCodec != NULL &&
1116                mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
1117                isAdaptive);
1118    }
1119
1120    ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
1121    return seamless;
1122}
1123
1124void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
1125    if (format == NULL) {
1126        return;
1127    }
1128    mCSDsForCurrentFormat.clear();
1129    for (int32_t i = 0; ; ++i) {
1130        AString tag = "csd-";
1131        tag.append(i);
1132        sp<ABuffer> buffer;
1133        if (!format->findBuffer(tag.c_str(), &buffer)) {
1134            break;
1135        }
1136        mCSDsForCurrentFormat.push(buffer);
1137    }
1138}
1139
1140void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
1141    if (mResumePending) {
1142        mResumePending = false;
1143
1144        sp<AMessage> notify = mNotify->dup();
1145        notify->setInt32("what", kWhatResumeCompleted);
1146        notify->post();
1147    }
1148}
1149
1150}  // namespace android
1151
1152