NuPlayerDecoder.cpp revision d72ffc4416959d2c94838888c781468b2351716d 
1/*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerDecoder"
19#include <utils/Log.h>
20#include <inttypes.h>
21
22#include <algorithm>
23
24#include "NuPlayerCCDecoder.h"
25#include "NuPlayerDecoder.h"
26#include "NuPlayerRenderer.h"
27#include "NuPlayerSource.h"
28
29#include <cutils/properties.h>
30#include <media/ICrypto.h>
31#include <media/MediaCodecBuffer.h>
32#include <media/stagefright/foundation/ABuffer.h>
33#include <media/stagefright/foundation/ADebug.h>
34#include <media/stagefright/foundation/AMessage.h>
35#include <media/stagefright/MediaBuffer.h>
36#include <media/stagefright/MediaCodec.h>
37#include <media/stagefright/MediaDefs.h>
38#include <media/stagefright/MediaErrors.h>
39
40#include <gui/Surface.h>
41
42#include "avc_utils.h"
43#include "ATSParser.h"
44
45namespace android {
46
47static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display
48
49// The default total video frame rate of a stream when that info is not available from
50// the source.
51static float kDefaultVideoFrameRateTotal = 30.f;
52
53static inline bool getAudioDeepBufferSetting() {
54    return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
55}
56
57NuPlayer::Decoder::Decoder(
58        const sp<AMessage> &notify,
59        const sp<Source> &source,
60        pid_t pid,
61        uid_t uid,
62        const sp<Renderer> &renderer,
63        const sp<Surface> &surface,
64        const sp<CCDecoder> &ccDecoder)
65    : DecoderBase(notify),
66      mSurface(surface),
67      mSource(source),
68      mRenderer(renderer),
69      mCCDecoder(ccDecoder),
70      mPid(pid),
71      mUid(uid),
72      mSkipRenderingUntilMediaTimeUs(-1ll),
73      mNumFramesTotal(0ll),
74      mNumInputFramesDropped(0ll),
75      mNumOutputFramesDropped(0ll),
76      mVideoWidth(0),
77      mVideoHeight(0),
78      mIsAudio(true),
79      mIsVideoAVC(false),
80      mIsSecure(false),
81      mFormatChangePending(false),
82      mTimeChangePending(false),
83      mFrameRateTotal(kDefaultVideoFrameRateTotal),
84      mPlaybackSpeed(1.0f),
85      mNumVideoTemporalLayerTotal(1), // decode all layers
86      mNumVideoTemporalLayerAllowed(1),
87      mCurrentMaxVideoTemporalLayerId(0),
88      mResumePending(false),
89      mComponentName("decoder") {
90    mCodecLooper = new ALooper;
91    mCodecLooper->setName("NPDecoder-CL");
92    mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
93    mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;
94}
95
96NuPlayer::Decoder::~Decoder() {
97    mCodec->release();
98    releaseAndResetMediaBuffers();
99}
100
101sp<AMessage> NuPlayer::Decoder::getStats() const {
102    mStats->setInt64("frames-total", mNumFramesTotal);
103    mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
104    mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
105    return mStats;
106}
107
108status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
109    if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
110        return BAD_VALUE;
111    }
112
113    sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
114
115    msg->setObject("surface", surface);
116    sp<AMessage> response;
117    status_t err = msg->postAndAwaitResponse(&response);
118    if (err == OK && response != NULL) {
119        CHECK(response->findInt32("err", &err));
120    }
121    return err;
122}
123
124void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
125    ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
126
127    switch (msg->what()) {
128        case kWhatCodecNotify:
129        {
130            int32_t cbID;
131            CHECK(msg->findInt32("callbackID", &cbID));
132
133            ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
134                    mIsAudio ? "audio" : "video", cbID, mPaused);
135
136            if (mPaused) {
137                break;
138            }
139
140            switch (cbID) {
141                case MediaCodec::CB_INPUT_AVAILABLE:
142                {
143                    int32_t index;
144                    CHECK(msg->findInt32("index", &index));
145
146                    handleAnInputBuffer(index);
147                    break;
148                }
149
150                case MediaCodec::CB_OUTPUT_AVAILABLE:
151                {
152                    int32_t index;
153                    size_t offset;
154                    size_t size;
155                    int64_t timeUs;
156                    int32_t flags;
157
158                    CHECK(msg->findInt32("index", &index));
159                    CHECK(msg->findSize("offset", &offset));
160                    CHECK(msg->findSize("size", &size));
161                    CHECK(msg->findInt64("timeUs", &timeUs));
162                    CHECK(msg->findInt32("flags", &flags));
163
164                    handleAnOutputBuffer(index, offset, size, timeUs, flags);
165                    break;
166                }
167
168                case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
169                {
170                    sp<AMessage> format;
171                    CHECK(msg->findMessage("format", &format));
172
173                    handleOutputFormatChange(format);
174                    break;
175                }
176
177                case MediaCodec::CB_ERROR:
178                {
179                    status_t err;
180                    CHECK(msg->findInt32("err", &err));
181                    ALOGE("Decoder (%s) reported error : 0x%x",
182                            mIsAudio ? "audio" : "video", err);
183
184                    handleError(err);
185                    break;
186                }
187
188                default:
189                {
190                    TRESPASS();
191                    break;
192                }
193            }
194
195            break;
196        }
197
198        case kWhatRenderBuffer:
199        {
200            if (!isStaleReply(msg)) {
201                onRenderBuffer(msg);
202            }
203            break;
204        }
205
206        case kWhatSetVideoSurface:
207        {
208            sp<AReplyToken> replyID;
209            CHECK(msg->senderAwaitsResponse(&replyID));
210
211            sp<RefBase> obj;
212            CHECK(msg->findObject("surface", &obj));
213            sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
214            int32_t err = INVALID_OPERATION;
215            // NOTE: in practice mSurface is always non-null, but checking here for completeness
216            if (mCodec != NULL && mSurface != NULL) {
217                // TODO: once AwesomePlayer is removed, remove this automatic connecting
218                // to the surface by MediaPlayerService.
219                //
220                // at this point MediaPlayerService::client has already connected to the
221                // surface, which MediaCodec does not expect
222                err = native_window_api_disconnect(surface.get(), NATIVE_WINDOW_API_MEDIA);
223                if (err == OK) {
224                    err = mCodec->setSurface(surface);
225                    ALOGI_IF(err, "codec setSurface returned: %d", err);
226                    if (err == OK) {
227                        // reconnect to the old surface as MPS::Client will expect to
228                        // be able to disconnect from it.
229                        (void)native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
230                        mSurface = surface;
231                    }
232                }
233                if (err != OK) {
234                    // reconnect to the new surface on error as MPS::Client will expect to
235                    // be able to disconnect from it.
236                    (void)native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA);
237                }
238            }
239
240            sp<AMessage> response = new AMessage;
241            response->setInt32("err", err);
242            response->postReply(replyID);
243            break;
244        }
245
246        default:
247            DecoderBase::onMessageReceived(msg);
248            break;
249    }
250}
251
252void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
253    CHECK(mCodec == NULL);
254
255    mFormatChangePending = false;
256    mTimeChangePending = false;
257
258    ++mBufferGeneration;
259
260    AString mime;
261    CHECK(format->findString("mime", &mime));
262
263    mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
264    mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
265
266    mComponentName = mime;
267    mComponentName.append(" decoder");
268    ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
269
270    mCodec = MediaCodec::CreateByType(
271            mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid);
272    int32_t secure = 0;
273    if (format->findInt32("secure", &secure) && secure != 0) {
274        if (mCodec != NULL) {
275            mCodec->getName(&mComponentName);
276            mComponentName.append(".secure");
277            mCodec->release();
278            ALOGI("[%s] creating", mComponentName.c_str());
279            mCodec = MediaCodec::CreateByComponentName(
280                    mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid);
281        }
282    }
283    if (mCodec == NULL) {
284        ALOGE("Failed to create %s%s decoder",
285                (secure ? "secure " : ""), mime.c_str());
286        handleError(UNKNOWN_ERROR);
287        return;
288    }
289    mIsSecure = secure;
290
291    mCodec->getName(&mComponentName);
292
293    status_t err;
294    if (mSurface != NULL) {
295        // disconnect from surface as MediaCodec will reconnect
296        err = native_window_api_disconnect(
297                mSurface.get(), NATIVE_WINDOW_API_MEDIA);
298        // We treat this as a warning, as this is a preparatory step.
299        // Codec will try to connect to the surface, which is where
300        // any error signaling will occur.
301        ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
302    }
303    err = mCodec->configure(
304            format, mSurface, NULL /* crypto */, 0 /* flags */);
305    if (err != OK) {
306        ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
307        mCodec->release();
308        mCodec.clear();
309        handleError(err);
310        return;
311    }
312    rememberCodecSpecificData(format);
313
314    // the following should work in configured state
315    CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
316    CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
317
318    mStats->setString("mime", mime.c_str());
319    mStats->setString("component-name", mComponentName.c_str());
320
321    if (!mIsAudio) {
322        int32_t width, height;
323        if (mOutputFormat->findInt32("width", &width)
324                && mOutputFormat->findInt32("height", &height)) {
325            mStats->setInt32("width", width);
326            mStats->setInt32("height", height);
327        }
328    }
329
330    sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
331    mCodec->setCallback(reply);
332
333    err = mCodec->start();
334    if (err != OK) {
335        ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
336        mCodec->release();
337        mCodec.clear();
338        handleError(err);
339        return;
340    }
341
342    releaseAndResetMediaBuffers();
343
344    mPaused = false;
345    mResumePending = false;
346}
347
348void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) {
349    bool needAdjustLayers = false;
350    float frameRateTotal;
351    if (params->findFloat("frame-rate-total", &frameRateTotal)
352            && mFrameRateTotal != frameRateTotal) {
353        needAdjustLayers = true;
354        mFrameRateTotal = frameRateTotal;
355    }
356
357    int32_t numVideoTemporalLayerTotal;
358    if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal)
359            && numVideoTemporalLayerTotal >= 0
360            && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers
361            && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) {
362        needAdjustLayers = true;
363        mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1);
364    }
365
366    if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) {
367        // TODO: For now, layer fps is calculated for some specific architectures.
368        // But it really should be extracted from the stream.
369        mVideoTemporalLayerAggregateFps[0] =
370            mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1));
371        for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) {
372            mVideoTemporalLayerAggregateFps[i] =
373                mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i))
374                + mVideoTemporalLayerAggregateFps[i - 1];
375        }
376    }
377
378    float playbackSpeed;
379    if (params->findFloat("playback-speed", &playbackSpeed)
380            && mPlaybackSpeed != playbackSpeed) {
381        needAdjustLayers = true;
382        mPlaybackSpeed = playbackSpeed;
383    }
384
385    if (needAdjustLayers) {
386        float decodeFrameRate = mFrameRateTotal;
387        // enable temporal layering optimization only if we know the layering depth
388        if (mNumVideoTemporalLayerTotal > 1) {
389            int32_t layerId;
390            for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) {
391                if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed
392                        >= kDisplayRefreshingRate * 0.9) {
393                    break;
394                }
395            }
396            mNumVideoTemporalLayerAllowed = layerId + 1;
397            decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId];
398        }
399        ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g",
400                mNumVideoTemporalLayerAllowed, decodeFrameRate);
401
402        if (mCodec == NULL) {
403            ALOGW("onSetParameters called before codec is created.");
404            return;
405        }
406
407        sp<AMessage> codecParams = new AMessage();
408        codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed);
409        mCodec->setParameters(codecParams);
410    }
411}
412
413void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
414    bool hadNoRenderer = (mRenderer == NULL);
415    mRenderer = renderer;
416    if (hadNoRenderer && mRenderer != NULL) {
417        // this means that the widevine legacy source is ready
418        onRequestInputBuffers();
419    }
420}
421
422void NuPlayer::Decoder::onGetInputBuffers(
423        Vector<sp<MediaCodecBuffer> > *dstBuffers) {
424    CHECK_EQ((status_t)OK, mCodec->getWidevineLegacyBuffers(dstBuffers));
425}
426
427void NuPlayer::Decoder::onResume(bool notifyComplete) {
428    mPaused = false;
429
430    if (notifyComplete) {
431        mResumePending = true;
432    }
433    mCodec->start();
434}
435
436void NuPlayer::Decoder::doFlush(bool notifyComplete) {
437    if (mCCDecoder != NULL) {
438        mCCDecoder->flush();
439    }
440
441    if (mRenderer != NULL) {
442        mRenderer->flush(mIsAudio, notifyComplete);
443        mRenderer->signalTimeDiscontinuity();
444    }
445
446    status_t err = OK;
447    if (mCodec != NULL) {
448        err = mCodec->flush();
449        mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
450        ++mBufferGeneration;
451    }
452
453    if (err != OK) {
454        ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
455        handleError(err);
456        // finish with posting kWhatFlushCompleted.
457        // we attempt to release the buffers even if flush fails.
458    }
459    releaseAndResetMediaBuffers();
460    mPaused = true;
461}
462
463
464void NuPlayer::Decoder::onFlush() {
465    doFlush(true);
466
467    if (isDiscontinuityPending()) {
468        // This could happen if the client starts seeking/shutdown
469        // after we queued an EOS for discontinuities.
470        // We can consider discontinuity handled.
471        finishHandleDiscontinuity(false /* flushOnTimeChange */);
472    }
473
474    sp<AMessage> notify = mNotify->dup();
475    notify->setInt32("what", kWhatFlushCompleted);
476    notify->post();
477}
478
479void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
480    status_t err = OK;
481
482    // if there is a pending resume request, notify complete now
483    notifyResumeCompleteIfNecessary();
484
485    if (mCodec != NULL) {
486        err = mCodec->release();
487        mCodec = NULL;
488        ++mBufferGeneration;
489
490        if (mSurface != NULL) {
491            // reconnect to surface as MediaCodec disconnected from it
492            status_t error =
493                    native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
494            ALOGW_IF(error != NO_ERROR,
495                    "[%s] failed to connect to native window, error=%d",
496                    mComponentName.c_str(), error);
497        }
498        mComponentName = "decoder";
499    }
500
501    releaseAndResetMediaBuffers();
502
503    if (err != OK) {
504        ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
505        handleError(err);
506        // finish with posting kWhatShutdownCompleted.
507    }
508
509    if (notifyComplete) {
510        sp<AMessage> notify = mNotify->dup();
511        notify->setInt32("what", kWhatShutdownCompleted);
512        notify->post();
513        mPaused = true;
514    }
515}
516
517/*
518 * returns true if we should request more data
519 */
520bool NuPlayer::Decoder::doRequestBuffers() {
521    // mRenderer is only NULL if we have a legacy widevine source that
522    // is not yet ready. In this case we must not fetch input.
523    if (isDiscontinuityPending() || mRenderer == NULL) {
524        return false;
525    }
526    status_t err = OK;
527    while (err == OK && !mDequeuedInputBuffers.empty()) {
528        size_t bufferIx = *mDequeuedInputBuffers.begin();
529        sp<AMessage> msg = new AMessage();
530        msg->setSize("buffer-ix", bufferIx);
531        err = fetchInputData(msg);
532        if (err != OK && err != ERROR_END_OF_STREAM) {
533            // if EOS, need to queue EOS buffer
534            break;
535        }
536        mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
537
538        if (!mPendingInputMessages.empty()
539                || !onInputBufferFetched(msg)) {
540            mPendingInputMessages.push_back(msg);
541        }
542    }
543
544    return err == -EWOULDBLOCK
545            && mSource->feedMoreTSData() == OK;
546}
547
548void NuPlayer::Decoder::handleError(int32_t err)
549{
550    // We cannot immediately release the codec due to buffers still outstanding
551    // in the renderer.  We signal to the player the error so it can shutdown/release the
552    // decoder after flushing and increment the generation to discard unnecessary messages.
553
554    ++mBufferGeneration;
555
556    sp<AMessage> notify = mNotify->dup();
557    notify->setInt32("what", kWhatError);
558    notify->setInt32("err", err);
559    notify->post();
560}
561
562bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
563    if (isDiscontinuityPending()) {
564        return false;
565    }
566
567    sp<MediaCodecBuffer> buffer;
568    mCodec->getInputBuffer(index, &buffer);
569
570    if (buffer == NULL) {
571        handleError(UNKNOWN_ERROR);
572        return false;
573    }
574
575    if (index >= mInputBuffers.size()) {
576        for (size_t i = mInputBuffers.size(); i <= index; ++i) {
577            mInputBuffers.add();
578            mMediaBuffers.add();
579            mInputBufferIsDequeued.add();
580            mMediaBuffers.editItemAt(i) = NULL;
581            mInputBufferIsDequeued.editItemAt(i) = false;
582        }
583    }
584    mInputBuffers.editItemAt(index) = buffer;
585
586    //CHECK_LT(bufferIx, mInputBuffers.size());
587
588    if (mMediaBuffers[index] != NULL) {
589        mMediaBuffers[index]->release();
590        mMediaBuffers.editItemAt(index) = NULL;
591    }
592    mInputBufferIsDequeued.editItemAt(index) = true;
593
594    if (!mCSDsToSubmit.isEmpty()) {
595        sp<AMessage> msg = new AMessage();
596        msg->setSize("buffer-ix", index);
597
598        sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
599        ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
600        msg->setBuffer("buffer", buffer);
601        mCSDsToSubmit.removeAt(0);
602        if (!onInputBufferFetched(msg)) {
603            handleError(UNKNOWN_ERROR);
604            return false;
605        }
606        return true;
607    }
608
609    while (!mPendingInputMessages.empty()) {
610        sp<AMessage> msg = *mPendingInputMessages.begin();
611        if (!onInputBufferFetched(msg)) {
612            break;
613        }
614        mPendingInputMessages.erase(mPendingInputMessages.begin());
615    }
616
617    if (!mInputBufferIsDequeued.editItemAt(index)) {
618        return true;
619    }
620
621    mDequeuedInputBuffers.push_back(index);
622
623    onRequestInputBuffers();
624    return true;
625}
626
627bool NuPlayer::Decoder::handleAnOutputBuffer(
628        size_t index,
629        size_t offset,
630        size_t size,
631        int64_t timeUs,
632        int32_t flags) {
633//    CHECK_LT(bufferIx, mOutputBuffers.size());
634    sp<MediaCodecBuffer> buffer;
635    mCodec->getOutputBuffer(index, &buffer);
636
637    if (index >= mOutputBuffers.size()) {
638        for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
639            mOutputBuffers.add();
640        }
641    }
642
643    mOutputBuffers.editItemAt(index) = buffer;
644
645    buffer->setRange(offset, size);
646    buffer->meta()->clear();
647    buffer->meta()->setInt64("timeUs", timeUs);
648
649    bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
650    // we do not expect CODECCONFIG or SYNCFRAME for decoder
651
652    sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
653    reply->setSize("buffer-ix", index);
654    reply->setInt32("generation", mBufferGeneration);
655
656    if (eos) {
657        ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
658
659        buffer->meta()->setInt32("eos", true);
660        reply->setInt32("eos", true);
661    } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
662        if (timeUs < mSkipRenderingUntilMediaTimeUs) {
663            ALOGV("[%s] dropping buffer at time %lld as requested.",
664                     mComponentName.c_str(), (long long)timeUs);
665
666            reply->post();
667            return true;
668        }
669
670        mSkipRenderingUntilMediaTimeUs = -1;
671    }
672
673    mNumFramesTotal += !mIsAudio;
674
675    // wait until 1st frame comes out to signal resume complete
676    notifyResumeCompleteIfNecessary();
677
678    if (mRenderer != NULL) {
679        // send the buffer to renderer.
680        mRenderer->queueBuffer(mIsAudio, buffer, reply);
681        if (eos && !isDiscontinuityPending()) {
682            mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
683        }
684    }
685
686    return true;
687}
688
689void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
690    if (!mIsAudio) {
691        int32_t width, height;
692        if (format->findInt32("width", &width)
693                && format->findInt32("height", &height)) {
694            mStats->setInt32("width", width);
695            mStats->setInt32("height", height);
696        }
697        sp<AMessage> notify = mNotify->dup();
698        notify->setInt32("what", kWhatVideoSizeChanged);
699        notify->setMessage("format", format);
700        notify->post();
701    } else if (mRenderer != NULL) {
702        uint32_t flags;
703        int64_t durationUs;
704        bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
705        if (getAudioDeepBufferSetting() // override regardless of source duration
706                || (!hasVideo
707                        && mSource->getDuration(&durationUs) == OK
708                        && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
709            flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
710        } else {
711            flags = AUDIO_OUTPUT_FLAG_NONE;
712        }
713
714        status_t err = mRenderer->openAudioSink(
715                format, false /* offloadOnly */, hasVideo, flags, NULL /* isOffloaed */);
716        if (err != OK) {
717            handleError(err);
718        }
719    }
720}
721
722void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
723    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
724        if (mMediaBuffers[i] != NULL) {
725            mMediaBuffers[i]->release();
726            mMediaBuffers.editItemAt(i) = NULL;
727        }
728    }
729    mMediaBuffers.resize(mInputBuffers.size());
730    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
731        mMediaBuffers.editItemAt(i) = NULL;
732    }
733    mInputBufferIsDequeued.clear();
734    mInputBufferIsDequeued.resize(mInputBuffers.size());
735    for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
736        mInputBufferIsDequeued.editItemAt(i) = false;
737    }
738
739    mPendingInputMessages.clear();
740    mDequeuedInputBuffers.clear();
741    mSkipRenderingUntilMediaTimeUs = -1;
742}
743
744void NuPlayer::Decoder::requestCodecNotification() {
745    if (mCodec != NULL) {
746        sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
747        reply->setInt32("generation", mBufferGeneration);
748        mCodec->requestActivityNotification(reply);
749    }
750}
751
752bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
753    int32_t generation;
754    CHECK(msg->findInt32("generation", &generation));
755    return generation != mBufferGeneration;
756}
757
758status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
759    sp<ABuffer> accessUnit;
760    bool dropAccessUnit = true;
761    do {
762        status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
763
764        if (err == -EWOULDBLOCK) {
765            return err;
766        } else if (err != OK) {
767            if (err == INFO_DISCONTINUITY) {
768                int32_t type;
769                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
770
771                bool formatChange =
772                    (mIsAudio &&
773                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
774                    || (!mIsAudio &&
775                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
776
777                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
778
779                ALOGI("%s discontinuity (format=%d, time=%d)",
780                        mIsAudio ? "audio" : "video", formatChange, timeChange);
781
782                bool seamlessFormatChange = false;
783                sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
784                if (formatChange) {
785                    seamlessFormatChange =
786                        supportsSeamlessFormatChange(newFormat);
787                    // treat seamless format change separately
788                    formatChange = !seamlessFormatChange;
789                }
790
791                // For format or time change, return EOS to queue EOS input,
792                // then wait for EOS on output.
793                if (formatChange /* not seamless */) {
794                    mFormatChangePending = true;
795                    err = ERROR_END_OF_STREAM;
796                } else if (timeChange) {
797                    rememberCodecSpecificData(newFormat);
798                    mTimeChangePending = true;
799                    err = ERROR_END_OF_STREAM;
800                } else if (seamlessFormatChange) {
801                    // reuse existing decoder and don't flush
802                    rememberCodecSpecificData(newFormat);
803                    continue;
804                } else {
805                    // This stream is unaffected by the discontinuity
806                    return -EWOULDBLOCK;
807                }
808            }
809
810            // reply should only be returned without a buffer set
811            // when there is an error (including EOS)
812            CHECK(err != OK);
813
814            reply->setInt32("err", err);
815            return ERROR_END_OF_STREAM;
816        }
817
818        dropAccessUnit = false;
819        if (!mIsAudio && !mIsSecure) {
820            int32_t layerId = 0;
821            bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId);
822            if (mRenderer->getVideoLateByUs() > 100000ll
823                    && mIsVideoAVC
824                    && !IsAVCReferenceFrame(accessUnit)) {
825                dropAccessUnit = true;
826            } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) {
827                // Add only one layer each time.
828                if (layerId > mCurrentMaxVideoTemporalLayerId + 1
829                        || layerId >= mNumVideoTemporalLayerAllowed) {
830                    dropAccessUnit = true;
831                    ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d",
832                            layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed,
833                            mCurrentMaxVideoTemporalLayerId);
834                } else if (layerId > mCurrentMaxVideoTemporalLayerId) {
835                    mCurrentMaxVideoTemporalLayerId = layerId;
836                } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) {
837                    mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1;
838                }
839            }
840            if (dropAccessUnit) {
841                if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) {
842                    mCurrentMaxVideoTemporalLayerId = layerId - 1;
843                }
844                ++mNumInputFramesDropped;
845            }
846        }
847    } while (dropAccessUnit);
848
849    // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
850#if 0
851    int64_t mediaTimeUs;
852    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
853    ALOGV("[%s] feeding input buffer at media time %.3f",
854         mIsAudio ? "audio" : "video",
855         mediaTimeUs / 1E6);
856#endif
857
858    if (mCCDecoder != NULL) {
859        mCCDecoder->decode(accessUnit);
860    }
861
862    reply->setBuffer("buffer", accessUnit);
863
864    return OK;
865}
866
867bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
868    size_t bufferIx;
869    CHECK(msg->findSize("buffer-ix", &bufferIx));
870    CHECK_LT(bufferIx, mInputBuffers.size());
871    sp<MediaCodecBuffer> codecBuffer = mInputBuffers[bufferIx];
872
873    sp<ABuffer> buffer;
874    bool hasBuffer = msg->findBuffer("buffer", &buffer);
875    bool needsCopy = true;
876
877    // handle widevine classic source - that fills an arbitrary input buffer
878    MediaBuffer *mediaBuffer = NULL;
879    if (hasBuffer) {
880        mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase());
881        if (mediaBuffer != NULL) {
882            // likely filled another buffer than we requested: adjust buffer index
883            size_t ix;
884            for (ix = 0; ix < mInputBuffers.size(); ix++) {
885                const sp<MediaCodecBuffer> &buf = mInputBuffers[ix];
886                if (buf->data() == mediaBuffer->data()) {
887                    // all input buffers are dequeued on start, hence the check
888                    if (!mInputBufferIsDequeued[ix]) {
889                        ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu",
890                                mComponentName.c_str(), ix, bufferIx);
891                        mediaBuffer->release();
892                        return false;
893                    }
894
895                    // TRICKY: need buffer for the metadata, so instead, set
896                    // codecBuffer to the same (though incorrect) buffer to
897                    // avoid a memcpy into the codecBuffer
898                    codecBuffer = new MediaCodecBuffer(codecBuffer->format(), buffer);
899                    codecBuffer->setRange(
900                            mediaBuffer->range_offset(),
901                            mediaBuffer->range_length());
902                    bufferIx = ix;
903                    needsCopy = false;
904                    break;
905                }
906            }
907            CHECK(ix < mInputBuffers.size());
908        }
909    }
910
911    if (buffer == NULL /* includes !hasBuffer */) {
912        int32_t streamErr = ERROR_END_OF_STREAM;
913        CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
914
915        CHECK(streamErr != OK);
916
917        // attempt to queue EOS
918        status_t err = mCodec->queueInputBuffer(
919                bufferIx,
920                0,
921                0,
922                0,
923                MediaCodec::BUFFER_FLAG_EOS);
924        if (err == OK) {
925            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
926        } else if (streamErr == ERROR_END_OF_STREAM) {
927            streamErr = err;
928            // err will not be ERROR_END_OF_STREAM
929        }
930
931        if (streamErr != ERROR_END_OF_STREAM) {
932            ALOGE("Stream error for %s (err=%d), EOS %s queued",
933                    mComponentName.c_str(),
934                    streamErr,
935                    err == OK ? "successfully" : "unsuccessfully");
936            handleError(streamErr);
937        }
938    } else {
939        sp<AMessage> extra;
940        if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
941            int64_t resumeAtMediaTimeUs;
942            if (extra->findInt64(
943                        "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
944                ALOGI("[%s] suppressing rendering until %lld us",
945                        mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
946                mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
947            }
948        }
949
950        int64_t timeUs = 0;
951        uint32_t flags = 0;
952        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
953
954        int32_t eos, csd;
955        // we do not expect SYNCFRAME for decoder
956        if (buffer->meta()->findInt32("eos", &eos) && eos) {
957            flags |= MediaCodec::BUFFER_FLAG_EOS;
958        } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
959            flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
960        }
961
962        // copy into codec buffer
963        if (needsCopy) {
964            if (buffer->size() > codecBuffer->capacity()) {
965                handleError(ERROR_BUFFER_TOO_SMALL);
966                mDequeuedInputBuffers.push_back(bufferIx);
967                return false;
968            }
969            codecBuffer->setRange(0, buffer->size());
970            memcpy(codecBuffer->data(), buffer->data(), buffer->size());
971        }
972
973        status_t err = mCodec->queueInputBuffer(
974                        bufferIx,
975                        codecBuffer->offset(),
976                        codecBuffer->size(),
977                        timeUs,
978                        flags);
979        if (err != OK) {
980            if (mediaBuffer != NULL) {
981                mediaBuffer->release();
982            }
983            ALOGE("Failed to queue input buffer for %s (err=%d)",
984                    mComponentName.c_str(), err);
985            handleError(err);
986        } else {
987            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
988            if (mediaBuffer != NULL) {
989                CHECK(mMediaBuffers[bufferIx] == NULL);
990                mMediaBuffers.editItemAt(bufferIx) = mediaBuffer;
991            }
992        }
993    }
994    return true;
995}
996
997void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
998    status_t err;
999    int32_t render;
1000    size_t bufferIx;
1001    int32_t eos;
1002    CHECK(msg->findSize("buffer-ix", &bufferIx));
1003
1004    if (!mIsAudio) {
1005        int64_t timeUs;
1006        sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx];
1007        buffer->meta()->findInt64("timeUs", &timeUs);
1008
1009        if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
1010            mCCDecoder->display(timeUs);
1011        }
1012    }
1013
1014    if (msg->findInt32("render", &render) && render) {
1015        int64_t timestampNs;
1016        CHECK(msg->findInt64("timestampNs", &timestampNs));
1017        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
1018    } else {
1019        mNumOutputFramesDropped += !mIsAudio;
1020        err = mCodec->releaseOutputBuffer(bufferIx);
1021    }
1022    if (err != OK) {
1023        ALOGE("failed to release output buffer for %s (err=%d)",
1024                mComponentName.c_str(), err);
1025        handleError(err);
1026    }
1027    if (msg->findInt32("eos", &eos) && eos
1028            && isDiscontinuityPending()) {
1029        finishHandleDiscontinuity(true /* flushOnTimeChange */);
1030    }
1031}
1032
1033bool NuPlayer::Decoder::isDiscontinuityPending() const {
1034    return mFormatChangePending || mTimeChangePending;
1035}
1036
1037void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
1038    ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
1039            mFormatChangePending, mTimeChangePending, flushOnTimeChange);
1040
1041    // If we have format change, pause and wait to be killed;
1042    // If we have time change only, flush and restart fetching.
1043
1044    if (mFormatChangePending) {
1045        mPaused = true;
1046    } else if (mTimeChangePending) {
1047        if (flushOnTimeChange) {
1048            doFlush(false /* notifyComplete */);
1049            signalResume(false /* notifyComplete */);
1050        }
1051    }
1052
1053    // Notify NuPlayer to either shutdown decoder, or rescan sources
1054    sp<AMessage> msg = mNotify->dup();
1055    msg->setInt32("what", kWhatInputDiscontinuity);
1056    msg->setInt32("formatChange", mFormatChangePending);
1057    msg->post();
1058
1059    mFormatChangePending = false;
1060    mTimeChangePending = false;
1061}
1062
1063bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
1064        const sp<AMessage> &targetFormat) const {
1065    if (targetFormat == NULL) {
1066        return true;
1067    }
1068
1069    AString mime;
1070    if (!targetFormat->findString("mime", &mime)) {
1071        return false;
1072    }
1073
1074    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
1075        // field-by-field comparison
1076        const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
1077        for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
1078            int32_t oldVal, newVal;
1079            if (!mInputFormat->findInt32(keys[i], &oldVal) ||
1080                    !targetFormat->findInt32(keys[i], &newVal) ||
1081                    oldVal != newVal) {
1082                return false;
1083            }
1084        }
1085
1086        sp<ABuffer> oldBuf, newBuf;
1087        if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
1088                targetFormat->findBuffer("csd-0", &newBuf)) {
1089            if (oldBuf->size() != newBuf->size()) {
1090                return false;
1091            }
1092            return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
1093        }
1094    }
1095    return false;
1096}
1097
1098bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
1099    if (mInputFormat == NULL) {
1100        return false;
1101    }
1102
1103    if (targetFormat == NULL) {
1104        return true;
1105    }
1106
1107    AString oldMime, newMime;
1108    if (!mInputFormat->findString("mime", &oldMime)
1109            || !targetFormat->findString("mime", &newMime)
1110            || !(oldMime == newMime)) {
1111        return false;
1112    }
1113
1114    bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
1115    bool seamless;
1116    if (audio) {
1117        seamless = supportsSeamlessAudioFormatChange(targetFormat);
1118    } else {
1119        int32_t isAdaptive;
1120        seamless = (mCodec != NULL &&
1121                mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
1122                isAdaptive);
1123    }
1124
1125    ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
1126    return seamless;
1127}
1128
1129void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
1130    if (format == NULL) {
1131        return;
1132    }
1133    mCSDsForCurrentFormat.clear();
1134    for (int32_t i = 0; ; ++i) {
1135        AString tag = "csd-";
1136        tag.append(i);
1137        sp<ABuffer> buffer;
1138        if (!format->findBuffer(tag.c_str(), &buffer)) {
1139            break;
1140        }
1141        mCSDsForCurrentFormat.push(buffer);
1142    }
1143}
1144
1145void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
1146    if (mResumePending) {
1147        mResumePending = false;
1148
1149        sp<AMessage> notify = mNotify->dup();
1150        notify->setInt32("what", kWhatResumeCompleted);
1151        notify->post();
1152    }
1153}
1154
1155}  // namespace android
1156
1157