NuPlayerDecoder.cpp revision 7e34bf5af26f8752d4786d3098740cdf51e2438f 
1/*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerDecoder"
19#include <utils/Log.h>
20#include <inttypes.h>
21
22#include <algorithm>
23
24#include "NuPlayerCCDecoder.h"
25#include "NuPlayerDecoder.h"
26#include "NuPlayerRenderer.h"
27#include "NuPlayerSource.h"
28
29#include <cutils/properties.h>
30#include <media/ICrypto.h>
31#include <media/MediaCodecBuffer.h>
32#include <media/stagefright/foundation/ABuffer.h>
33#include <media/stagefright/foundation/ADebug.h>
34#include <media/stagefright/foundation/AMessage.h>
35#include <media/stagefright/MediaBuffer.h>
36#include <media/stagefright/MediaCodec.h>
37#include <media/stagefright/MediaDefs.h>
38#include <media/stagefright/MediaErrors.h>
39
40#include <gui/Surface.h>
41
42#include "avc_utils.h"
43#include "ATSParser.h"
44
45namespace android {
46
47static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display
48
49// The default total video frame rate of a stream when that info is not available from
50// the source.
51static float kDefaultVideoFrameRateTotal = 30.f;
52
53static inline bool getAudioDeepBufferSetting() {
54    return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
55}
56
57NuPlayer::Decoder::Decoder(
58        const sp<AMessage> &notify,
59        const sp<Source> &source,
60        pid_t pid,
61        const sp<Renderer> &renderer,
62        const sp<Surface> &surface,
63        const sp<CCDecoder> &ccDecoder)
64    : DecoderBase(notify),
65      mSurface(surface),
66      mSource(source),
67      mRenderer(renderer),
68      mCCDecoder(ccDecoder),
69      mPid(pid),
70      mSkipRenderingUntilMediaTimeUs(-1ll),
71      mNumFramesTotal(0ll),
72      mNumInputFramesDropped(0ll),
73      mNumOutputFramesDropped(0ll),
74      mVideoWidth(0),
75      mVideoHeight(0),
76      mIsAudio(true),
77      mIsVideoAVC(false),
78      mIsSecure(false),
79      mFormatChangePending(false),
80      mTimeChangePending(false),
81      mFrameRateTotal(kDefaultVideoFrameRateTotal),
82      mPlaybackSpeed(1.0f),
83      mNumVideoTemporalLayerTotal(1), // decode all layers
84      mNumVideoTemporalLayerAllowed(1),
85      mCurrentMaxVideoTemporalLayerId(0),
86      mResumePending(false),
87      mComponentName("decoder") {
88    mCodecLooper = new ALooper;
89    mCodecLooper->setName("NPDecoder-CL");
90    mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
91    mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;
92}
93
94NuPlayer::Decoder::~Decoder() {
95    mCodec->release();
96    releaseAndResetMediaBuffers();
97}
98
99sp<AMessage> NuPlayer::Decoder::getStats() const {
100    mStats->setInt64("frames-total", mNumFramesTotal);
101    mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
102    mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
103    return mStats;
104}
105
106status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
107    if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
108        return BAD_VALUE;
109    }
110
111    sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
112
113    msg->setObject("surface", surface);
114    sp<AMessage> response;
115    status_t err = msg->postAndAwaitResponse(&response);
116    if (err == OK && response != NULL) {
117        CHECK(response->findInt32("err", &err));
118    }
119    return err;
120}
121
122void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
123    ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
124
125    switch (msg->what()) {
126        case kWhatCodecNotify:
127        {
128            int32_t cbID;
129            CHECK(msg->findInt32("callbackID", &cbID));
130
131            ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
132                    mIsAudio ? "audio" : "video", cbID, mPaused);
133
134            if (mPaused) {
135                break;
136            }
137
138            switch (cbID) {
139                case MediaCodec::CB_INPUT_AVAILABLE:
140                {
141                    int32_t index;
142                    CHECK(msg->findInt32("index", &index));
143
144                    handleAnInputBuffer(index);
145                    break;
146                }
147
148                case MediaCodec::CB_OUTPUT_AVAILABLE:
149                {
150                    int32_t index;
151                    size_t offset;
152                    size_t size;
153                    int64_t timeUs;
154                    int32_t flags;
155
156                    CHECK(msg->findInt32("index", &index));
157                    CHECK(msg->findSize("offset", &offset));
158                    CHECK(msg->findSize("size", &size));
159                    CHECK(msg->findInt64("timeUs", &timeUs));
160                    CHECK(msg->findInt32("flags", &flags));
161
162                    handleAnOutputBuffer(index, offset, size, timeUs, flags);
163                    break;
164                }
165
166                case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
167                {
168                    sp<AMessage> format;
169                    CHECK(msg->findMessage("format", &format));
170
171                    handleOutputFormatChange(format);
172                    break;
173                }
174
175                case MediaCodec::CB_ERROR:
176                {
177                    status_t err;
178                    CHECK(msg->findInt32("err", &err));
179                    ALOGE("Decoder (%s) reported error : 0x%x",
180                            mIsAudio ? "audio" : "video", err);
181
182                    handleError(err);
183                    break;
184                }
185
186                default:
187                {
188                    TRESPASS();
189                    break;
190                }
191            }
192
193            break;
194        }
195
196        case kWhatRenderBuffer:
197        {
198            if (!isStaleReply(msg)) {
199                onRenderBuffer(msg);
200            }
201            break;
202        }
203
204        case kWhatSetVideoSurface:
205        {
206            sp<AReplyToken> replyID;
207            CHECK(msg->senderAwaitsResponse(&replyID));
208
209            sp<RefBase> obj;
210            CHECK(msg->findObject("surface", &obj));
211            sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
212            int32_t err = INVALID_OPERATION;
213            // NOTE: in practice mSurface is always non-null, but checking here for completeness
214            if (mCodec != NULL && mSurface != NULL) {
215                // TODO: once AwesomePlayer is removed, remove this automatic connecting
216                // to the surface by MediaPlayerService.
217                //
218                // at this point MediaPlayerService::client has already connected to the
219                // surface, which MediaCodec does not expect
220                err = native_window_api_disconnect(surface.get(), NATIVE_WINDOW_API_MEDIA);
221                if (err == OK) {
222                    err = mCodec->setSurface(surface);
223                    ALOGI_IF(err, "codec setSurface returned: %d", err);
224                    if (err == OK) {
225                        // reconnect to the old surface as MPS::Client will expect to
226                        // be able to disconnect from it.
227                        (void)native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
228                        mSurface = surface;
229                    }
230                }
231                if (err != OK) {
232                    // reconnect to the new surface on error as MPS::Client will expect to
233                    // be able to disconnect from it.
234                    (void)native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA);
235                }
236            }
237
238            sp<AMessage> response = new AMessage;
239            response->setInt32("err", err);
240            response->postReply(replyID);
241            break;
242        }
243
244        default:
245            DecoderBase::onMessageReceived(msg);
246            break;
247    }
248}
249
250void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
251    CHECK(mCodec == NULL);
252
253    mFormatChangePending = false;
254    mTimeChangePending = false;
255
256    ++mBufferGeneration;
257
258    AString mime;
259    CHECK(format->findString("mime", &mime));
260
261    mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
262    mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
263
264    mComponentName = mime;
265    mComponentName.append(" decoder");
266    ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
267
268    mCodec = MediaCodec::CreateByType(
269            mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid);
270    int32_t secure = 0;
271    if (format->findInt32("secure", &secure) && secure != 0) {
272        if (mCodec != NULL) {
273            mCodec->getName(&mComponentName);
274            mComponentName.append(".secure");
275            mCodec->release();
276            ALOGI("[%s] creating", mComponentName.c_str());
277            mCodec = MediaCodec::CreateByComponentName(
278                    mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid);
279        }
280    }
281    if (mCodec == NULL) {
282        ALOGE("Failed to create %s%s decoder",
283                (secure ? "secure " : ""), mime.c_str());
284        handleError(UNKNOWN_ERROR);
285        return;
286    }
287    mIsSecure = secure;
288
289    mCodec->getName(&mComponentName);
290
291    status_t err;
292    if (mSurface != NULL) {
293        // disconnect from surface as MediaCodec will reconnect
294        err = native_window_api_disconnect(
295                mSurface.get(), NATIVE_WINDOW_API_MEDIA);
296        // We treat this as a warning, as this is a preparatory step.
297        // Codec will try to connect to the surface, which is where
298        // any error signaling will occur.
299        ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
300    }
301    err = mCodec->configure(
302            format, mSurface, NULL /* crypto */, 0 /* flags */);
303    if (err != OK) {
304        ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
305        mCodec->release();
306        mCodec.clear();
307        handleError(err);
308        return;
309    }
310    rememberCodecSpecificData(format);
311
312    // the following should work in configured state
313    CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
314    CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
315
316    mStats->setString("mime", mime.c_str());
317    mStats->setString("component-name", mComponentName.c_str());
318
319    if (!mIsAudio) {
320        int32_t width, height;
321        if (mOutputFormat->findInt32("width", &width)
322                && mOutputFormat->findInt32("height", &height)) {
323            mStats->setInt32("width", width);
324            mStats->setInt32("height", height);
325        }
326    }
327
328    sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
329    mCodec->setCallback(reply);
330
331    err = mCodec->start();
332    if (err != OK) {
333        ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
334        mCodec->release();
335        mCodec.clear();
336        handleError(err);
337        return;
338    }
339
340    releaseAndResetMediaBuffers();
341
342    mPaused = false;
343    mResumePending = false;
344}
345
346void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) {
347    bool needAdjustLayers = false;
348    float frameRateTotal;
349    if (params->findFloat("frame-rate-total", &frameRateTotal)
350            && mFrameRateTotal != frameRateTotal) {
351        needAdjustLayers = true;
352        mFrameRateTotal = frameRateTotal;
353    }
354
355    int32_t numVideoTemporalLayerTotal;
356    if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal)
357            && numVideoTemporalLayerTotal >= 0
358            && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers
359            && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) {
360        needAdjustLayers = true;
361        mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1);
362    }
363
364    if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) {
365        // TODO: For now, layer fps is calculated for some specific architectures.
366        // But it really should be extracted from the stream.
367        mVideoTemporalLayerAggregateFps[0] =
368            mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1));
369        for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) {
370            mVideoTemporalLayerAggregateFps[i] =
371                mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i))
372                + mVideoTemporalLayerAggregateFps[i - 1];
373        }
374    }
375
376    float playbackSpeed;
377    if (params->findFloat("playback-speed", &playbackSpeed)
378            && mPlaybackSpeed != playbackSpeed) {
379        needAdjustLayers = true;
380        mPlaybackSpeed = playbackSpeed;
381    }
382
383    if (needAdjustLayers) {
384        float decodeFrameRate = mFrameRateTotal;
385        // enable temporal layering optimization only if we know the layering depth
386        if (mNumVideoTemporalLayerTotal > 1) {
387            int32_t layerId;
388            for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) {
389                if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed
390                        >= kDisplayRefreshingRate * 0.9) {
391                    break;
392                }
393            }
394            mNumVideoTemporalLayerAllowed = layerId + 1;
395            decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId];
396        }
397        ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g",
398                mNumVideoTemporalLayerAllowed, decodeFrameRate);
399
400        if (mCodec == NULL) {
401            ALOGW("onSetParameters called before codec is created.");
402            return;
403        }
404
405        sp<AMessage> codecParams = new AMessage();
406        codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed);
407        mCodec->setParameters(codecParams);
408    }
409}
410
411void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
412    bool hadNoRenderer = (mRenderer == NULL);
413    mRenderer = renderer;
414    if (hadNoRenderer && mRenderer != NULL) {
415        // this means that the widevine legacy source is ready
416        onRequestInputBuffers();
417    }
418}
419
420void NuPlayer::Decoder::onGetInputBuffers(
421        Vector<sp<MediaCodecBuffer> > *dstBuffers) {
422    CHECK_EQ((status_t)OK, mCodec->getWidevineLegacyBuffers(dstBuffers));
423}
424
425void NuPlayer::Decoder::onResume(bool notifyComplete) {
426    mPaused = false;
427
428    if (notifyComplete) {
429        mResumePending = true;
430    }
431    mCodec->start();
432}
433
434void NuPlayer::Decoder::doFlush(bool notifyComplete) {
435    if (mCCDecoder != NULL) {
436        mCCDecoder->flush();
437    }
438
439    if (mRenderer != NULL) {
440        mRenderer->flush(mIsAudio, notifyComplete);
441        mRenderer->signalTimeDiscontinuity();
442    }
443
444    status_t err = OK;
445    if (mCodec != NULL) {
446        err = mCodec->flush();
447        mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
448        ++mBufferGeneration;
449    }
450
451    if (err != OK) {
452        ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
453        handleError(err);
454        // finish with posting kWhatFlushCompleted.
455        // we attempt to release the buffers even if flush fails.
456    }
457    releaseAndResetMediaBuffers();
458    mPaused = true;
459}
460
461
462void NuPlayer::Decoder::onFlush() {
463    doFlush(true);
464
465    if (isDiscontinuityPending()) {
466        // This could happen if the client starts seeking/shutdown
467        // after we queued an EOS for discontinuities.
468        // We can consider discontinuity handled.
469        finishHandleDiscontinuity(false /* flushOnTimeChange */);
470    }
471
472    sp<AMessage> notify = mNotify->dup();
473    notify->setInt32("what", kWhatFlushCompleted);
474    notify->post();
475}
476
477void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
478    status_t err = OK;
479
480    // if there is a pending resume request, notify complete now
481    notifyResumeCompleteIfNecessary();
482
483    if (mCodec != NULL) {
484        err = mCodec->release();
485        mCodec = NULL;
486        ++mBufferGeneration;
487
488        if (mSurface != NULL) {
489            // reconnect to surface as MediaCodec disconnected from it
490            status_t error =
491                    native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
492            ALOGW_IF(error != NO_ERROR,
493                    "[%s] failed to connect to native window, error=%d",
494                    mComponentName.c_str(), error);
495        }
496        mComponentName = "decoder";
497    }
498
499    releaseAndResetMediaBuffers();
500
501    if (err != OK) {
502        ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
503        handleError(err);
504        // finish with posting kWhatShutdownCompleted.
505    }
506
507    if (notifyComplete) {
508        sp<AMessage> notify = mNotify->dup();
509        notify->setInt32("what", kWhatShutdownCompleted);
510        notify->post();
511        mPaused = true;
512    }
513}
514
515/*
516 * returns true if we should request more data
517 */
518bool NuPlayer::Decoder::doRequestBuffers() {
519    // mRenderer is only NULL if we have a legacy widevine source that
520    // is not yet ready. In this case we must not fetch input.
521    if (isDiscontinuityPending() || mRenderer == NULL) {
522        return false;
523    }
524    status_t err = OK;
525    while (err == OK && !mDequeuedInputBuffers.empty()) {
526        size_t bufferIx = *mDequeuedInputBuffers.begin();
527        sp<AMessage> msg = new AMessage();
528        msg->setSize("buffer-ix", bufferIx);
529        err = fetchInputData(msg);
530        if (err != OK && err != ERROR_END_OF_STREAM) {
531            // if EOS, need to queue EOS buffer
532            break;
533        }
534        mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
535
536        if (!mPendingInputMessages.empty()
537                || !onInputBufferFetched(msg)) {
538            mPendingInputMessages.push_back(msg);
539        }
540    }
541
542    return err == -EWOULDBLOCK
543            && mSource->feedMoreTSData() == OK;
544}
545
546void NuPlayer::Decoder::handleError(int32_t err)
547{
548    // We cannot immediately release the codec due to buffers still outstanding
549    // in the renderer.  We signal to the player the error so it can shutdown/release the
550    // decoder after flushing and increment the generation to discard unnecessary messages.
551
552    ++mBufferGeneration;
553
554    sp<AMessage> notify = mNotify->dup();
555    notify->setInt32("what", kWhatError);
556    notify->setInt32("err", err);
557    notify->post();
558}
559
560bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
561    if (isDiscontinuityPending()) {
562        return false;
563    }
564
565    sp<MediaCodecBuffer> buffer;
566    mCodec->getInputBuffer(index, &buffer);
567
568    if (buffer == NULL) {
569        handleError(UNKNOWN_ERROR);
570        return false;
571    }
572
573    if (index >= mInputBuffers.size()) {
574        for (size_t i = mInputBuffers.size(); i <= index; ++i) {
575            mInputBuffers.add();
576            mMediaBuffers.add();
577            mInputBufferIsDequeued.add();
578            mMediaBuffers.editItemAt(i) = NULL;
579            mInputBufferIsDequeued.editItemAt(i) = false;
580        }
581    }
582    mInputBuffers.editItemAt(index) = buffer;
583
584    //CHECK_LT(bufferIx, mInputBuffers.size());
585
586    if (mMediaBuffers[index] != NULL) {
587        mMediaBuffers[index]->release();
588        mMediaBuffers.editItemAt(index) = NULL;
589    }
590    mInputBufferIsDequeued.editItemAt(index) = true;
591
592    if (!mCSDsToSubmit.isEmpty()) {
593        sp<AMessage> msg = new AMessage();
594        msg->setSize("buffer-ix", index);
595
596        sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
597        ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
598        msg->setBuffer("buffer", buffer);
599        mCSDsToSubmit.removeAt(0);
600        if (!onInputBufferFetched(msg)) {
601            handleError(UNKNOWN_ERROR);
602            return false;
603        }
604        return true;
605    }
606
607    while (!mPendingInputMessages.empty()) {
608        sp<AMessage> msg = *mPendingInputMessages.begin();
609        if (!onInputBufferFetched(msg)) {
610            break;
611        }
612        mPendingInputMessages.erase(mPendingInputMessages.begin());
613    }
614
615    if (!mInputBufferIsDequeued.editItemAt(index)) {
616        return true;
617    }
618
619    mDequeuedInputBuffers.push_back(index);
620
621    onRequestInputBuffers();
622    return true;
623}
624
625bool NuPlayer::Decoder::handleAnOutputBuffer(
626        size_t index,
627        size_t offset,
628        size_t size,
629        int64_t timeUs,
630        int32_t flags) {
631//    CHECK_LT(bufferIx, mOutputBuffers.size());
632    sp<MediaCodecBuffer> buffer;
633    mCodec->getOutputBuffer(index, &buffer);
634
635    if (index >= mOutputBuffers.size()) {
636        for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
637            mOutputBuffers.add();
638        }
639    }
640
641    mOutputBuffers.editItemAt(index) = buffer;
642
643    buffer->setRange(offset, size);
644    buffer->meta()->clear();
645    buffer->meta()->setInt64("timeUs", timeUs);
646
647    bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
648    // we do not expect CODECCONFIG or SYNCFRAME for decoder
649
650    sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
651    reply->setSize("buffer-ix", index);
652    reply->setInt32("generation", mBufferGeneration);
653
654    if (eos) {
655        ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
656
657        buffer->meta()->setInt32("eos", true);
658        reply->setInt32("eos", true);
659    } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
660        if (timeUs < mSkipRenderingUntilMediaTimeUs) {
661            ALOGV("[%s] dropping buffer at time %lld as requested.",
662                     mComponentName.c_str(), (long long)timeUs);
663
664            reply->post();
665            return true;
666        }
667
668        mSkipRenderingUntilMediaTimeUs = -1;
669    }
670
671    mNumFramesTotal += !mIsAudio;
672
673    // wait until 1st frame comes out to signal resume complete
674    notifyResumeCompleteIfNecessary();
675
676    if (mRenderer != NULL) {
677        // send the buffer to renderer.
678        mRenderer->queueBuffer(mIsAudio, buffer, reply);
679        if (eos && !isDiscontinuityPending()) {
680            mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
681        }
682    }
683
684    return true;
685}
686
687void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
688    if (!mIsAudio) {
689        int32_t width, height;
690        if (format->findInt32("width", &width)
691                && format->findInt32("height", &height)) {
692            mStats->setInt32("width", width);
693            mStats->setInt32("height", height);
694        }
695        sp<AMessage> notify = mNotify->dup();
696        notify->setInt32("what", kWhatVideoSizeChanged);
697        notify->setMessage("format", format);
698        notify->post();
699    } else if (mRenderer != NULL) {
700        uint32_t flags;
701        int64_t durationUs;
702        bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
703        if (getAudioDeepBufferSetting() // override regardless of source duration
704                || (!hasVideo
705                        && mSource->getDuration(&durationUs) == OK
706                        && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
707            flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
708        } else {
709            flags = AUDIO_OUTPUT_FLAG_NONE;
710        }
711
712        status_t err = mRenderer->openAudioSink(
713                format, false /* offloadOnly */, hasVideo, flags, NULL /* isOffloaed */);
714        if (err != OK) {
715            handleError(err);
716        }
717    }
718}
719
720void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
721    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
722        if (mMediaBuffers[i] != NULL) {
723            mMediaBuffers[i]->release();
724            mMediaBuffers.editItemAt(i) = NULL;
725        }
726    }
727    mMediaBuffers.resize(mInputBuffers.size());
728    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
729        mMediaBuffers.editItemAt(i) = NULL;
730    }
731    mInputBufferIsDequeued.clear();
732    mInputBufferIsDequeued.resize(mInputBuffers.size());
733    for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
734        mInputBufferIsDequeued.editItemAt(i) = false;
735    }
736
737    mPendingInputMessages.clear();
738    mDequeuedInputBuffers.clear();
739    mSkipRenderingUntilMediaTimeUs = -1;
740}
741
742void NuPlayer::Decoder::requestCodecNotification() {
743    if (mCodec != NULL) {
744        sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
745        reply->setInt32("generation", mBufferGeneration);
746        mCodec->requestActivityNotification(reply);
747    }
748}
749
750bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
751    int32_t generation;
752    CHECK(msg->findInt32("generation", &generation));
753    return generation != mBufferGeneration;
754}
755
756status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
757    sp<ABuffer> accessUnit;
758    bool dropAccessUnit;
759    do {
760        status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
761
762        if (err == -EWOULDBLOCK) {
763            return err;
764        } else if (err != OK) {
765            if (err == INFO_DISCONTINUITY) {
766                int32_t type;
767                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
768
769                bool formatChange =
770                    (mIsAudio &&
771                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
772                    || (!mIsAudio &&
773                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
774
775                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
776
777                ALOGI("%s discontinuity (format=%d, time=%d)",
778                        mIsAudio ? "audio" : "video", formatChange, timeChange);
779
780                bool seamlessFormatChange = false;
781                sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
782                if (formatChange) {
783                    seamlessFormatChange =
784                        supportsSeamlessFormatChange(newFormat);
785                    // treat seamless format change separately
786                    formatChange = !seamlessFormatChange;
787                }
788
789                // For format or time change, return EOS to queue EOS input,
790                // then wait for EOS on output.
791                if (formatChange /* not seamless */) {
792                    mFormatChangePending = true;
793                    err = ERROR_END_OF_STREAM;
794                } else if (timeChange) {
795                    rememberCodecSpecificData(newFormat);
796                    mTimeChangePending = true;
797                    err = ERROR_END_OF_STREAM;
798                } else if (seamlessFormatChange) {
799                    // reuse existing decoder and don't flush
800                    rememberCodecSpecificData(newFormat);
801                    continue;
802                } else {
803                    // This stream is unaffected by the discontinuity
804                    return -EWOULDBLOCK;
805                }
806            }
807
808            // reply should only be returned without a buffer set
809            // when there is an error (including EOS)
810            CHECK(err != OK);
811
812            reply->setInt32("err", err);
813            return ERROR_END_OF_STREAM;
814        }
815
816        dropAccessUnit = false;
817        if (!mIsAudio && !mIsSecure) {
818            int32_t layerId = 0;
819            bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId);
820            if (mRenderer->getVideoLateByUs() > 100000ll
821                    && mIsVideoAVC
822                    && !IsAVCReferenceFrame(accessUnit)) {
823                dropAccessUnit = true;
824            } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) {
825                // Add only one layer each time.
826                if (layerId > mCurrentMaxVideoTemporalLayerId + 1
827                        || layerId >= mNumVideoTemporalLayerAllowed) {
828                    dropAccessUnit = true;
829                    ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d",
830                            layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed,
831                            mCurrentMaxVideoTemporalLayerId);
832                } else if (layerId > mCurrentMaxVideoTemporalLayerId) {
833                    mCurrentMaxVideoTemporalLayerId = layerId;
834                } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) {
835                    mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1;
836                }
837            }
838            if (dropAccessUnit) {
839                if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) {
840                    mCurrentMaxVideoTemporalLayerId = layerId - 1;
841                }
842                ++mNumInputFramesDropped;
843            }
844        }
845    } while (dropAccessUnit);
846
847    // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
848#if 0
849    int64_t mediaTimeUs;
850    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
851    ALOGV("[%s] feeding input buffer at media time %.3f",
852         mIsAudio ? "audio" : "video",
853         mediaTimeUs / 1E6);
854#endif
855
856    if (mCCDecoder != NULL) {
857        mCCDecoder->decode(accessUnit);
858    }
859
860    reply->setBuffer("buffer", accessUnit);
861
862    return OK;
863}
864
865bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
866    size_t bufferIx;
867    CHECK(msg->findSize("buffer-ix", &bufferIx));
868    CHECK_LT(bufferIx, mInputBuffers.size());
869    sp<MediaCodecBuffer> codecBuffer = mInputBuffers[bufferIx];
870
871    sp<ABuffer> buffer;
872    bool hasBuffer = msg->findBuffer("buffer", &buffer);
873    bool needsCopy = true;
874
875    // handle widevine classic source - that fills an arbitrary input buffer
876    MediaBuffer *mediaBuffer = NULL;
877    if (hasBuffer) {
878        mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase());
879        if (mediaBuffer != NULL) {
880            // likely filled another buffer than we requested: adjust buffer index
881            size_t ix;
882            for (ix = 0; ix < mInputBuffers.size(); ix++) {
883                const sp<MediaCodecBuffer> &buf = mInputBuffers[ix];
884                if (buf->data() == mediaBuffer->data()) {
885                    // all input buffers are dequeued on start, hence the check
886                    if (!mInputBufferIsDequeued[ix]) {
887                        ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu",
888                                mComponentName.c_str(), ix, bufferIx);
889                        mediaBuffer->release();
890                        return false;
891                    }
892
893                    // TRICKY: need buffer for the metadata, so instead, set
894                    // codecBuffer to the same (though incorrect) buffer to
895                    // avoid a memcpy into the codecBuffer
896                    codecBuffer = new MediaCodecBuffer(codecBuffer->format(), buffer);
897                    codecBuffer->setRange(
898                            mediaBuffer->range_offset(),
899                            mediaBuffer->range_length());
900                    bufferIx = ix;
901                    needsCopy = false;
902                    break;
903                }
904            }
905            CHECK(ix < mInputBuffers.size());
906        }
907    }
908
909    if (buffer == NULL /* includes !hasBuffer */) {
910        int32_t streamErr = ERROR_END_OF_STREAM;
911        CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
912
913        CHECK(streamErr != OK);
914
915        // attempt to queue EOS
916        status_t err = mCodec->queueInputBuffer(
917                bufferIx,
918                0,
919                0,
920                0,
921                MediaCodec::BUFFER_FLAG_EOS);
922        if (err == OK) {
923            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
924        } else if (streamErr == ERROR_END_OF_STREAM) {
925            streamErr = err;
926            // err will not be ERROR_END_OF_STREAM
927        }
928
929        if (streamErr != ERROR_END_OF_STREAM) {
930            ALOGE("Stream error for %s (err=%d), EOS %s queued",
931                    mComponentName.c_str(),
932                    streamErr,
933                    err == OK ? "successfully" : "unsuccessfully");
934            handleError(streamErr);
935        }
936    } else {
937        sp<AMessage> extra;
938        if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
939            int64_t resumeAtMediaTimeUs;
940            if (extra->findInt64(
941                        "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
942                ALOGI("[%s] suppressing rendering until %lld us",
943                        mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
944                mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
945            }
946        }
947
948        int64_t timeUs = 0;
949        uint32_t flags = 0;
950        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
951
952        int32_t eos, csd;
953        // we do not expect SYNCFRAME for decoder
954        if (buffer->meta()->findInt32("eos", &eos) && eos) {
955            flags |= MediaCodec::BUFFER_FLAG_EOS;
956        } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
957            flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
958        }
959
960        // copy into codec buffer
961        if (needsCopy) {
962            if (buffer->size() > codecBuffer->capacity()) {
963                handleError(ERROR_BUFFER_TOO_SMALL);
964                mDequeuedInputBuffers.push_back(bufferIx);
965                return false;
966            }
967            codecBuffer->setRange(0, buffer->size());
968            memcpy(codecBuffer->data(), buffer->data(), buffer->size());
969        }
970
971        status_t err = mCodec->queueInputBuffer(
972                        bufferIx,
973                        codecBuffer->offset(),
974                        codecBuffer->size(),
975                        timeUs,
976                        flags);
977        if (err != OK) {
978            if (mediaBuffer != NULL) {
979                mediaBuffer->release();
980            }
981            ALOGE("Failed to queue input buffer for %s (err=%d)",
982                    mComponentName.c_str(), err);
983            handleError(err);
984        } else {
985            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
986            if (mediaBuffer != NULL) {
987                CHECK(mMediaBuffers[bufferIx] == NULL);
988                mMediaBuffers.editItemAt(bufferIx) = mediaBuffer;
989            }
990        }
991    }
992    return true;
993}
994
995void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
996    status_t err;
997    int32_t render;
998    size_t bufferIx;
999    int32_t eos;
1000    CHECK(msg->findSize("buffer-ix", &bufferIx));
1001
1002    if (!mIsAudio) {
1003        int64_t timeUs;
1004        sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx];
1005        buffer->meta()->findInt64("timeUs", &timeUs);
1006
1007        if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
1008            mCCDecoder->display(timeUs);
1009        }
1010    }
1011
1012    if (msg->findInt32("render", &render) && render) {
1013        int64_t timestampNs;
1014        CHECK(msg->findInt64("timestampNs", &timestampNs));
1015        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
1016    } else {
1017        mNumOutputFramesDropped += !mIsAudio;
1018        err = mCodec->releaseOutputBuffer(bufferIx);
1019    }
1020    if (err != OK) {
1021        ALOGE("failed to release output buffer for %s (err=%d)",
1022                mComponentName.c_str(), err);
1023        handleError(err);
1024    }
1025    if (msg->findInt32("eos", &eos) && eos
1026            && isDiscontinuityPending()) {
1027        finishHandleDiscontinuity(true /* flushOnTimeChange */);
1028    }
1029}
1030
1031bool NuPlayer::Decoder::isDiscontinuityPending() const {
1032    return mFormatChangePending || mTimeChangePending;
1033}
1034
1035void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
1036    ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
1037            mFormatChangePending, mTimeChangePending, flushOnTimeChange);
1038
1039    // If we have format change, pause and wait to be killed;
1040    // If we have time change only, flush and restart fetching.
1041
1042    if (mFormatChangePending) {
1043        mPaused = true;
1044    } else if (mTimeChangePending) {
1045        if (flushOnTimeChange) {
1046            doFlush(false /* notifyComplete */);
1047            signalResume(false /* notifyComplete */);
1048        }
1049    }
1050
1051    // Notify NuPlayer to either shutdown decoder, or rescan sources
1052    sp<AMessage> msg = mNotify->dup();
1053    msg->setInt32("what", kWhatInputDiscontinuity);
1054    msg->setInt32("formatChange", mFormatChangePending);
1055    msg->post();
1056
1057    mFormatChangePending = false;
1058    mTimeChangePending = false;
1059}
1060
1061bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
1062        const sp<AMessage> &targetFormat) const {
1063    if (targetFormat == NULL) {
1064        return true;
1065    }
1066
1067    AString mime;
1068    if (!targetFormat->findString("mime", &mime)) {
1069        return false;
1070    }
1071
1072    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
1073        // field-by-field comparison
1074        const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
1075        for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
1076            int32_t oldVal, newVal;
1077            if (!mInputFormat->findInt32(keys[i], &oldVal) ||
1078                    !targetFormat->findInt32(keys[i], &newVal) ||
1079                    oldVal != newVal) {
1080                return false;
1081            }
1082        }
1083
1084        sp<ABuffer> oldBuf, newBuf;
1085        if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
1086                targetFormat->findBuffer("csd-0", &newBuf)) {
1087            if (oldBuf->size() != newBuf->size()) {
1088                return false;
1089            }
1090            return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
1091        }
1092    }
1093    return false;
1094}
1095
1096bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
1097    if (mInputFormat == NULL) {
1098        return false;
1099    }
1100
1101    if (targetFormat == NULL) {
1102        return true;
1103    }
1104
1105    AString oldMime, newMime;
1106    if (!mInputFormat->findString("mime", &oldMime)
1107            || !targetFormat->findString("mime", &newMime)
1108            || !(oldMime == newMime)) {
1109        return false;
1110    }
1111
1112    bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
1113    bool seamless;
1114    if (audio) {
1115        seamless = supportsSeamlessAudioFormatChange(targetFormat);
1116    } else {
1117        int32_t isAdaptive;
1118        seamless = (mCodec != NULL &&
1119                mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
1120                isAdaptive);
1121    }
1122
1123    ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
1124    return seamless;
1125}
1126
1127void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
1128    if (format == NULL) {
1129        return;
1130    }
1131    mCSDsForCurrentFormat.clear();
1132    for (int32_t i = 0; ; ++i) {
1133        AString tag = "csd-";
1134        tag.append(i);
1135        sp<ABuffer> buffer;
1136        if (!format->findBuffer(tag.c_str(), &buffer)) {
1137            break;
1138        }
1139        mCSDsForCurrentFormat.push(buffer);
1140    }
1141}
1142
1143void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
1144    if (mResumePending) {
1145        mResumePending = false;
1146
1147        sp<AMessage> notify = mNotify->dup();
1148        notify->setInt32("what", kWhatResumeCompleted);
1149        notify->post();
1150    }
1151}
1152
1153}  // namespace android
1154
1155