NuPlayerDecoder.cpp revision 37ff0e6639e90ca49d4f0386a76c09437b459efd 
1/*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerDecoder"
19#include <utils/Log.h>
20#include <inttypes.h>
21
22#include <algorithm>
23
24#include "NuPlayerCCDecoder.h"
25#include "NuPlayerDecoder.h"
26#include "NuPlayerDrm.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29
30#include <cutils/properties.h>
31#include <media/ICrypto.h>
32#include <media/MediaCodecBuffer.h>
33#include <media/stagefright/foundation/ABuffer.h>
34#include <media/stagefright/foundation/ADebug.h>
35#include <media/stagefright/foundation/AMessage.h>
36#include <media/stagefright/MediaBuffer.h>
37#include <media/stagefright/MediaCodec.h>
38#include <media/stagefright/MediaDefs.h>
39#include <media/stagefright/MediaErrors.h>
40#include <media/stagefright/SurfaceUtils.h>
41#include <gui/Surface.h>
42
43#include "avc_utils.h"
44#include "ATSParser.h"
45
46namespace android {
47
48static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display
49
50// The default total video frame rate of a stream when that info is not available from
51// the source.
52static float kDefaultVideoFrameRateTotal = 30.f;
53
54static inline bool getAudioDeepBufferSetting() {
55    return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
56}
57
58NuPlayer::Decoder::Decoder(
59        const sp<AMessage> &notify,
60        const sp<Source> &source,
61        pid_t pid,
62        uid_t uid,
63        const sp<Renderer> &renderer,
64        const sp<Surface> &surface,
65        const sp<CCDecoder> &ccDecoder)
66    : DecoderBase(notify),
67      mSurface(surface),
68      mSource(source),
69      mRenderer(renderer),
70      mCCDecoder(ccDecoder),
71      mPid(pid),
72      mUid(uid),
73      mSkipRenderingUntilMediaTimeUs(-1ll),
74      mNumFramesTotal(0ll),
75      mNumInputFramesDropped(0ll),
76      mNumOutputFramesDropped(0ll),
77      mVideoWidth(0),
78      mVideoHeight(0),
79      mIsAudio(true),
80      mIsVideoAVC(false),
81      mIsSecure(false),
82      mFormatChangePending(false),
83      mTimeChangePending(false),
84      mFrameRateTotal(kDefaultVideoFrameRateTotal),
85      mPlaybackSpeed(1.0f),
86      mNumVideoTemporalLayerTotal(1), // decode all layers
87      mNumVideoTemporalLayerAllowed(1),
88      mCurrentMaxVideoTemporalLayerId(0),
89      mResumePending(false),
90      mComponentName("decoder") {
91    mCodecLooper = new ALooper;
92    mCodecLooper->setName("NPDecoder-CL");
93    mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
94    mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;
95}
96
97NuPlayer::Decoder::~Decoder() {
98    // Need to stop looper first since mCodec could be accessed on the mDecoderLooper.
99    stopLooper();
100    if (mCodec != NULL) {
101        mCodec->release();
102    }
103    releaseAndResetMediaBuffers();
104}
105
106sp<AMessage> NuPlayer::Decoder::getStats() const {
107    mStats->setInt64("frames-total", mNumFramesTotal);
108    mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
109    mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
110    return mStats;
111}
112
113status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
114    if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
115        return BAD_VALUE;
116    }
117
118    sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
119
120    msg->setObject("surface", surface);
121    sp<AMessage> response;
122    status_t err = msg->postAndAwaitResponse(&response);
123    if (err == OK && response != NULL) {
124        CHECK(response->findInt32("err", &err));
125    }
126    return err;
127}
128
129void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
130    ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
131
132    switch (msg->what()) {
133        case kWhatCodecNotify:
134        {
135            int32_t cbID;
136            CHECK(msg->findInt32("callbackID", &cbID));
137
138            ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
139                    mIsAudio ? "audio" : "video", cbID, mPaused);
140
141            if (mPaused) {
142                break;
143            }
144
145            switch (cbID) {
146                case MediaCodec::CB_INPUT_AVAILABLE:
147                {
148                    int32_t index;
149                    CHECK(msg->findInt32("index", &index));
150
151                    handleAnInputBuffer(index);
152                    break;
153                }
154
155                case MediaCodec::CB_OUTPUT_AVAILABLE:
156                {
157                    int32_t index;
158                    size_t offset;
159                    size_t size;
160                    int64_t timeUs;
161                    int32_t flags;
162
163                    CHECK(msg->findInt32("index", &index));
164                    CHECK(msg->findSize("offset", &offset));
165                    CHECK(msg->findSize("size", &size));
166                    CHECK(msg->findInt64("timeUs", &timeUs));
167                    CHECK(msg->findInt32("flags", &flags));
168
169                    handleAnOutputBuffer(index, offset, size, timeUs, flags);
170                    break;
171                }
172
173                case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
174                {
175                    sp<AMessage> format;
176                    CHECK(msg->findMessage("format", &format));
177
178                    handleOutputFormatChange(format);
179                    break;
180                }
181
182                case MediaCodec::CB_ERROR:
183                {
184                    status_t err;
185                    CHECK(msg->findInt32("err", &err));
186                    ALOGE("Decoder (%s) reported error : 0x%x",
187                            mIsAudio ? "audio" : "video", err);
188
189                    handleError(err);
190                    break;
191                }
192
193                default:
194                {
195                    TRESPASS();
196                    break;
197                }
198            }
199
200            break;
201        }
202
203        case kWhatRenderBuffer:
204        {
205            if (!isStaleReply(msg)) {
206                onRenderBuffer(msg);
207            }
208            break;
209        }
210
211        case kWhatAudioOutputFormatChanged:
212        {
213            if (!isStaleReply(msg)) {
214                status_t err;
215                if (msg->findInt32("err", &err) && err != OK) {
216                    ALOGE("Renderer reported 0x%x when changing audio output format", err);
217                    handleError(err);
218                }
219            }
220            break;
221        }
222
223        case kWhatSetVideoSurface:
224        {
225            sp<AReplyToken> replyID;
226            CHECK(msg->senderAwaitsResponse(&replyID));
227
228            sp<RefBase> obj;
229            CHECK(msg->findObject("surface", &obj));
230            sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
231            int32_t err = INVALID_OPERATION;
232            // NOTE: in practice mSurface is always non-null, but checking here for completeness
233            if (mCodec != NULL && mSurface != NULL) {
234                // TODO: once AwesomePlayer is removed, remove this automatic connecting
235                // to the surface by MediaPlayerService.
236                //
237                // at this point MediaPlayerService::client has already connected to the
238                // surface, which MediaCodec does not expect
239                err = nativeWindowDisconnect(surface.get(), "kWhatSetVideoSurface(surface)");
240                if (err == OK) {
241                    err = mCodec->setSurface(surface);
242                    ALOGI_IF(err, "codec setSurface returned: %d", err);
243                    if (err == OK) {
244                        // reconnect to the old surface as MPS::Client will expect to
245                        // be able to disconnect from it.
246                        (void)nativeWindowConnect(mSurface.get(), "kWhatSetVideoSurface(mSurface)");
247                        mSurface = surface;
248                    }
249                }
250                if (err != OK) {
251                    // reconnect to the new surface on error as MPS::Client will expect to
252                    // be able to disconnect from it.
253                    (void)nativeWindowConnect(surface.get(), "kWhatSetVideoSurface(err)");
254                }
255            }
256
257            sp<AMessage> response = new AMessage;
258            response->setInt32("err", err);
259            response->postReply(replyID);
260            break;
261        }
262
263        case kWhatDrmReleaseCrypto:
264        {
265            ALOGV("kWhatDrmReleaseCrypto");
266            onReleaseCrypto(msg);
267            break;
268        }
269
270        default:
271            DecoderBase::onMessageReceived(msg);
272            break;
273    }
274}
275
276void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
277    CHECK(mCodec == NULL);
278
279    mFormatChangePending = false;
280    mTimeChangePending = false;
281
282    ++mBufferGeneration;
283
284    AString mime;
285    CHECK(format->findString("mime", &mime));
286
287    mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
288    mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
289
290    mComponentName = mime;
291    mComponentName.append(" decoder");
292    ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
293
294    mCodec = MediaCodec::CreateByType(
295            mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid);
296    int32_t secure = 0;
297    if (format->findInt32("secure", &secure) && secure != 0) {
298        if (mCodec != NULL) {
299            mCodec->getName(&mComponentName);
300            mComponentName.append(".secure");
301            mCodec->release();
302            ALOGI("[%s] creating", mComponentName.c_str());
303            mCodec = MediaCodec::CreateByComponentName(
304                    mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid);
305        }
306    }
307    if (mCodec == NULL) {
308        ALOGE("Failed to create %s%s decoder",
309                (secure ? "secure " : ""), mime.c_str());
310        handleError(UNKNOWN_ERROR);
311        return;
312    }
313    mIsSecure = secure;
314
315    mCodec->getName(&mComponentName);
316
317    status_t err;
318    if (mSurface != NULL) {
319        // disconnect from surface as MediaCodec will reconnect
320        err = nativeWindowDisconnect(mSurface.get(), "onConfigure");
321        // We treat this as a warning, as this is a preparatory step.
322        // Codec will try to connect to the surface, which is where
323        // any error signaling will occur.
324        ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
325    }
326
327    // Modular DRM
328    void *pCrypto;
329    if (!format->findPointer("crypto", &pCrypto)) {
330        pCrypto = NULL;
331    }
332    sp<ICrypto> crypto = (ICrypto*)pCrypto;
333    ALOGV("onConfigure mCrypto: %p (%d)  mIsSecure: %d",
334            crypto.get(), (crypto != NULL ? crypto->getStrongCount() : 0), mIsSecure);
335
336    err = mCodec->configure(
337            format, mSurface, crypto, 0 /* flags */);
338
339    if (err != OK) {
340        ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
341        mCodec->release();
342        mCodec.clear();
343        handleError(err);
344        return;
345    }
346    rememberCodecSpecificData(format);
347
348    // the following should work in configured state
349    CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
350    CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
351
352    mStats->setString("mime", mime.c_str());
353    mStats->setString("component-name", mComponentName.c_str());
354
355    if (!mIsAudio) {
356        int32_t width, height;
357        if (mOutputFormat->findInt32("width", &width)
358                && mOutputFormat->findInt32("height", &height)) {
359            mStats->setInt32("width", width);
360            mStats->setInt32("height", height);
361        }
362    }
363
364    sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
365    mCodec->setCallback(reply);
366
367    err = mCodec->start();
368    if (err != OK) {
369        ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
370        mCodec->release();
371        mCodec.clear();
372        handleError(err);
373        return;
374    }
375
376    releaseAndResetMediaBuffers();
377
378    mPaused = false;
379    mResumePending = false;
380}
381
382void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) {
383    bool needAdjustLayers = false;
384    float frameRateTotal;
385    if (params->findFloat("frame-rate-total", &frameRateTotal)
386            && mFrameRateTotal != frameRateTotal) {
387        needAdjustLayers = true;
388        mFrameRateTotal = frameRateTotal;
389    }
390
391    int32_t numVideoTemporalLayerTotal;
392    if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal)
393            && numVideoTemporalLayerTotal >= 0
394            && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers
395            && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) {
396        needAdjustLayers = true;
397        mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1);
398    }
399
400    if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) {
401        // TODO: For now, layer fps is calculated for some specific architectures.
402        // But it really should be extracted from the stream.
403        mVideoTemporalLayerAggregateFps[0] =
404            mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1));
405        for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) {
406            mVideoTemporalLayerAggregateFps[i] =
407                mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i))
408                + mVideoTemporalLayerAggregateFps[i - 1];
409        }
410    }
411
412    float playbackSpeed;
413    if (params->findFloat("playback-speed", &playbackSpeed)
414            && mPlaybackSpeed != playbackSpeed) {
415        needAdjustLayers = true;
416        mPlaybackSpeed = playbackSpeed;
417    }
418
419    if (needAdjustLayers) {
420        float decodeFrameRate = mFrameRateTotal;
421        // enable temporal layering optimization only if we know the layering depth
422        if (mNumVideoTemporalLayerTotal > 1) {
423            int32_t layerId;
424            for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) {
425                if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed
426                        >= kDisplayRefreshingRate * 0.9) {
427                    break;
428                }
429            }
430            mNumVideoTemporalLayerAllowed = layerId + 1;
431            decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId];
432        }
433        ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g",
434                mNumVideoTemporalLayerAllowed, decodeFrameRate);
435
436        if (mCodec == NULL) {
437            ALOGW("onSetParameters called before codec is created.");
438            return;
439        }
440
441        sp<AMessage> codecParams = new AMessage();
442        codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed);
443        mCodec->setParameters(codecParams);
444    }
445}
446
447void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
448    mRenderer = renderer;
449}
450
451void NuPlayer::Decoder::onResume(bool notifyComplete) {
452    mPaused = false;
453
454    if (notifyComplete) {
455        mResumePending = true;
456    }
457    mCodec->start();
458}
459
460void NuPlayer::Decoder::doFlush(bool notifyComplete) {
461    if (mCCDecoder != NULL) {
462        mCCDecoder->flush();
463    }
464
465    if (mRenderer != NULL) {
466        mRenderer->flush(mIsAudio, notifyComplete);
467        mRenderer->signalTimeDiscontinuity();
468    }
469
470    status_t err = OK;
471    if (mCodec != NULL) {
472        err = mCodec->flush();
473        mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
474        ++mBufferGeneration;
475    }
476
477    if (err != OK) {
478        ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
479        handleError(err);
480        // finish with posting kWhatFlushCompleted.
481        // we attempt to release the buffers even if flush fails.
482    }
483    releaseAndResetMediaBuffers();
484    mPaused = true;
485}
486
487
488void NuPlayer::Decoder::onFlush() {
489    doFlush(true);
490
491    if (isDiscontinuityPending()) {
492        // This could happen if the client starts seeking/shutdown
493        // after we queued an EOS for discontinuities.
494        // We can consider discontinuity handled.
495        finishHandleDiscontinuity(false /* flushOnTimeChange */);
496    }
497
498    sp<AMessage> notify = mNotify->dup();
499    notify->setInt32("what", kWhatFlushCompleted);
500    notify->post();
501}
502
503void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
504    status_t err = OK;
505
506    // if there is a pending resume request, notify complete now
507    notifyResumeCompleteIfNecessary();
508
509    if (mCodec != NULL) {
510        err = mCodec->release();
511        mCodec = NULL;
512        ++mBufferGeneration;
513
514        if (mSurface != NULL) {
515            // reconnect to surface as MediaCodec disconnected from it
516            status_t error = nativeWindowConnect(mSurface.get(), "onShutdown");
517            ALOGW_IF(error != NO_ERROR,
518                    "[%s] failed to connect to native window, error=%d",
519                    mComponentName.c_str(), error);
520        }
521        mComponentName = "decoder";
522    }
523
524    releaseAndResetMediaBuffers();
525
526    if (err != OK) {
527        ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
528        handleError(err);
529        // finish with posting kWhatShutdownCompleted.
530    }
531
532    if (notifyComplete) {
533        sp<AMessage> notify = mNotify->dup();
534        notify->setInt32("what", kWhatShutdownCompleted);
535        notify->post();
536        mPaused = true;
537    }
538}
539
540/*
541 * returns true if we should request more data
542 */
543bool NuPlayer::Decoder::doRequestBuffers() {
544    if (isDiscontinuityPending()) {
545        return false;
546    }
547    status_t err = OK;
548    while (err == OK && !mDequeuedInputBuffers.empty()) {
549        size_t bufferIx = *mDequeuedInputBuffers.begin();
550        sp<AMessage> msg = new AMessage();
551        msg->setSize("buffer-ix", bufferIx);
552        err = fetchInputData(msg);
553        if (err != OK && err != ERROR_END_OF_STREAM) {
554            // if EOS, need to queue EOS buffer
555            break;
556        }
557        mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
558
559        if (!mPendingInputMessages.empty()
560                || !onInputBufferFetched(msg)) {
561            mPendingInputMessages.push_back(msg);
562        }
563    }
564
565    return err == -EWOULDBLOCK
566            && mSource->feedMoreTSData() == OK;
567}
568
569void NuPlayer::Decoder::handleError(int32_t err)
570{
571    // We cannot immediately release the codec due to buffers still outstanding
572    // in the renderer.  We signal to the player the error so it can shutdown/release the
573    // decoder after flushing and increment the generation to discard unnecessary messages.
574
575    ++mBufferGeneration;
576
577    sp<AMessage> notify = mNotify->dup();
578    notify->setInt32("what", kWhatError);
579    notify->setInt32("err", err);
580    notify->post();
581}
582
583status_t NuPlayer::Decoder::releaseCrypto()
584{
585    ALOGV("releaseCrypto");
586
587    sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this);
588
589    sp<AMessage> response;
590    status_t status = msg->postAndAwaitResponse(&response);
591    if (status == OK && response != NULL) {
592        CHECK(response->findInt32("status", &status));
593        ALOGV("releaseCrypto ret: %d ", status);
594    } else {
595        ALOGE("releaseCrypto err: %d", status);
596    }
597
598    return status;
599}
600
601void NuPlayer::Decoder::onReleaseCrypto(const sp<AMessage>& msg)
602{
603    status_t status = INVALID_OPERATION;
604    if (mCodec != NULL) {
605        status = mCodec->releaseCrypto();
606    } else {
607        // returning OK if the codec has been already released
608        status = OK;
609        ALOGE("onReleaseCrypto No mCodec. err: %d", status);
610    }
611
612    sp<AMessage> response = new AMessage;
613    response->setInt32("status", status);
614
615    sp<AReplyToken> replyID;
616    CHECK(msg->senderAwaitsResponse(&replyID));
617    response->postReply(replyID);
618}
619
620bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
621    if (isDiscontinuityPending()) {
622        return false;
623    }
624
625    sp<MediaCodecBuffer> buffer;
626    mCodec->getInputBuffer(index, &buffer);
627
628    if (buffer == NULL) {
629        handleError(UNKNOWN_ERROR);
630        return false;
631    }
632
633    if (index >= mInputBuffers.size()) {
634        for (size_t i = mInputBuffers.size(); i <= index; ++i) {
635            mInputBuffers.add();
636            mMediaBuffers.add();
637            mInputBufferIsDequeued.add();
638            mMediaBuffers.editItemAt(i) = NULL;
639            mInputBufferIsDequeued.editItemAt(i) = false;
640        }
641    }
642    mInputBuffers.editItemAt(index) = buffer;
643
644    //CHECK_LT(bufferIx, mInputBuffers.size());
645
646    if (mMediaBuffers[index] != NULL) {
647        mMediaBuffers[index]->release();
648        mMediaBuffers.editItemAt(index) = NULL;
649    }
650    mInputBufferIsDequeued.editItemAt(index) = true;
651
652    if (!mCSDsToSubmit.isEmpty()) {
653        sp<AMessage> msg = new AMessage();
654        msg->setSize("buffer-ix", index);
655
656        sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
657        ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
658        msg->setBuffer("buffer", buffer);
659        mCSDsToSubmit.removeAt(0);
660        if (!onInputBufferFetched(msg)) {
661            handleError(UNKNOWN_ERROR);
662            return false;
663        }
664        return true;
665    }
666
667    while (!mPendingInputMessages.empty()) {
668        sp<AMessage> msg = *mPendingInputMessages.begin();
669        if (!onInputBufferFetched(msg)) {
670            break;
671        }
672        mPendingInputMessages.erase(mPendingInputMessages.begin());
673    }
674
675    if (!mInputBufferIsDequeued.editItemAt(index)) {
676        return true;
677    }
678
679    mDequeuedInputBuffers.push_back(index);
680
681    onRequestInputBuffers();
682    return true;
683}
684
685bool NuPlayer::Decoder::handleAnOutputBuffer(
686        size_t index,
687        size_t offset,
688        size_t size,
689        int64_t timeUs,
690        int32_t flags) {
691//    CHECK_LT(bufferIx, mOutputBuffers.size());
692    sp<MediaCodecBuffer> buffer;
693    mCodec->getOutputBuffer(index, &buffer);
694
695    if (buffer == NULL) {
696        handleError(UNKNOWN_ERROR);
697        return false;
698    }
699
700    if (index >= mOutputBuffers.size()) {
701        for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
702            mOutputBuffers.add();
703        }
704    }
705
706    mOutputBuffers.editItemAt(index) = buffer;
707
708    buffer->setRange(offset, size);
709    buffer->meta()->clear();
710    buffer->meta()->setInt64("timeUs", timeUs);
711
712    bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
713    // we do not expect CODECCONFIG or SYNCFRAME for decoder
714
715    sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
716    reply->setSize("buffer-ix", index);
717    reply->setInt32("generation", mBufferGeneration);
718
719    if (eos) {
720        ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
721
722        buffer->meta()->setInt32("eos", true);
723        reply->setInt32("eos", true);
724    } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
725        if (timeUs < mSkipRenderingUntilMediaTimeUs) {
726            ALOGV("[%s] dropping buffer at time %lld as requested.",
727                     mComponentName.c_str(), (long long)timeUs);
728
729            reply->post();
730            return true;
731        }
732
733        mSkipRenderingUntilMediaTimeUs = -1;
734    }
735
736    mNumFramesTotal += !mIsAudio;
737
738    // wait until 1st frame comes out to signal resume complete
739    notifyResumeCompleteIfNecessary();
740
741    if (mRenderer != NULL) {
742        // send the buffer to renderer.
743        mRenderer->queueBuffer(mIsAudio, buffer, reply);
744        if (eos && !isDiscontinuityPending()) {
745            mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
746        }
747    }
748
749    return true;
750}
751
752void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
753    if (!mIsAudio) {
754        int32_t width, height;
755        if (format->findInt32("width", &width)
756                && format->findInt32("height", &height)) {
757            mStats->setInt32("width", width);
758            mStats->setInt32("height", height);
759        }
760        sp<AMessage> notify = mNotify->dup();
761        notify->setInt32("what", kWhatVideoSizeChanged);
762        notify->setMessage("format", format);
763        notify->post();
764    } else if (mRenderer != NULL) {
765        uint32_t flags;
766        int64_t durationUs;
767        bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
768        if (getAudioDeepBufferSetting() // override regardless of source duration
769                || (mSource->getDuration(&durationUs) == OK
770                        && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
771            flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
772        } else {
773            flags = AUDIO_OUTPUT_FLAG_NONE;
774        }
775
776        sp<AMessage> reply = new AMessage(kWhatAudioOutputFormatChanged, this);
777        reply->setInt32("generation", mBufferGeneration);
778        mRenderer->changeAudioFormat(
779                format, false /* offloadOnly */, hasVideo,
780                flags, mSource->isStreaming(), reply);
781    }
782}
783
784void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
785    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
786        if (mMediaBuffers[i] != NULL) {
787            mMediaBuffers[i]->release();
788            mMediaBuffers.editItemAt(i) = NULL;
789        }
790    }
791    mMediaBuffers.resize(mInputBuffers.size());
792    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
793        mMediaBuffers.editItemAt(i) = NULL;
794    }
795    mInputBufferIsDequeued.clear();
796    mInputBufferIsDequeued.resize(mInputBuffers.size());
797    for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
798        mInputBufferIsDequeued.editItemAt(i) = false;
799    }
800
801    mPendingInputMessages.clear();
802    mDequeuedInputBuffers.clear();
803    mSkipRenderingUntilMediaTimeUs = -1;
804}
805
806void NuPlayer::Decoder::requestCodecNotification() {
807    if (mCodec != NULL) {
808        sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
809        reply->setInt32("generation", mBufferGeneration);
810        mCodec->requestActivityNotification(reply);
811    }
812}
813
814bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
815    int32_t generation;
816    CHECK(msg->findInt32("generation", &generation));
817    return generation != mBufferGeneration;
818}
819
820status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
821    sp<ABuffer> accessUnit;
822    bool dropAccessUnit = true;
823    do {
824        status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
825
826        if (err == -EWOULDBLOCK) {
827            return err;
828        } else if (err != OK) {
829            if (err == INFO_DISCONTINUITY) {
830                int32_t type;
831                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
832
833                bool formatChange =
834                    (mIsAudio &&
835                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
836                    || (!mIsAudio &&
837                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
838
839                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
840
841                ALOGI("%s discontinuity (format=%d, time=%d)",
842                        mIsAudio ? "audio" : "video", formatChange, timeChange);
843
844                bool seamlessFormatChange = false;
845                sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
846                if (formatChange) {
847                    seamlessFormatChange =
848                        supportsSeamlessFormatChange(newFormat);
849                    // treat seamless format change separately
850                    formatChange = !seamlessFormatChange;
851                }
852
853                // For format or time change, return EOS to queue EOS input,
854                // then wait for EOS on output.
855                if (formatChange /* not seamless */) {
856                    mFormatChangePending = true;
857                    err = ERROR_END_OF_STREAM;
858                } else if (timeChange) {
859                    rememberCodecSpecificData(newFormat);
860                    mTimeChangePending = true;
861                    err = ERROR_END_OF_STREAM;
862                } else if (seamlessFormatChange) {
863                    // reuse existing decoder and don't flush
864                    rememberCodecSpecificData(newFormat);
865                    continue;
866                } else {
867                    // This stream is unaffected by the discontinuity
868                    return -EWOULDBLOCK;
869                }
870            }
871
872            // reply should only be returned without a buffer set
873            // when there is an error (including EOS)
874            CHECK(err != OK);
875
876            reply->setInt32("err", err);
877            return ERROR_END_OF_STREAM;
878        }
879
880        dropAccessUnit = false;
881        if (!mIsAudio && !mIsSecure) {
882            int32_t layerId = 0;
883            bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId);
884            if (mRenderer->getVideoLateByUs() > 100000ll
885                    && mIsVideoAVC
886                    && !IsAVCReferenceFrame(accessUnit)) {
887                dropAccessUnit = true;
888            } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) {
889                // Add only one layer each time.
890                if (layerId > mCurrentMaxVideoTemporalLayerId + 1
891                        || layerId >= mNumVideoTemporalLayerAllowed) {
892                    dropAccessUnit = true;
893                    ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d",
894                            layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed,
895                            mCurrentMaxVideoTemporalLayerId);
896                } else if (layerId > mCurrentMaxVideoTemporalLayerId) {
897                    mCurrentMaxVideoTemporalLayerId = layerId;
898                } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) {
899                    mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1;
900                }
901            }
902            if (dropAccessUnit) {
903                if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) {
904                    mCurrentMaxVideoTemporalLayerId = layerId - 1;
905                }
906                ++mNumInputFramesDropped;
907            }
908        }
909    } while (dropAccessUnit);
910
911    // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
912#if 0
913    int64_t mediaTimeUs;
914    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
915    ALOGV("[%s] feeding input buffer at media time %.3f",
916         mIsAudio ? "audio" : "video",
917         mediaTimeUs / 1E6);
918#endif
919
920    if (mCCDecoder != NULL) {
921        mCCDecoder->decode(accessUnit);
922    }
923
924    reply->setBuffer("buffer", accessUnit);
925
926    return OK;
927}
928
929bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
930    size_t bufferIx;
931    CHECK(msg->findSize("buffer-ix", &bufferIx));
932    CHECK_LT(bufferIx, mInputBuffers.size());
933    sp<MediaCodecBuffer> codecBuffer = mInputBuffers[bufferIx];
934
935    sp<ABuffer> buffer;
936    bool hasBuffer = msg->findBuffer("buffer", &buffer);
937    bool needsCopy = true;
938
939    if (buffer == NULL /* includes !hasBuffer */) {
940        int32_t streamErr = ERROR_END_OF_STREAM;
941        CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
942
943        CHECK(streamErr != OK);
944
945        // attempt to queue EOS
946        status_t err = mCodec->queueInputBuffer(
947                bufferIx,
948                0,
949                0,
950                0,
951                MediaCodec::BUFFER_FLAG_EOS);
952        if (err == OK) {
953            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
954        } else if (streamErr == ERROR_END_OF_STREAM) {
955            streamErr = err;
956            // err will not be ERROR_END_OF_STREAM
957        }
958
959        if (streamErr != ERROR_END_OF_STREAM) {
960            ALOGE("Stream error for %s (err=%d), EOS %s queued",
961                    mComponentName.c_str(),
962                    streamErr,
963                    err == OK ? "successfully" : "unsuccessfully");
964            handleError(streamErr);
965        }
966    } else {
967        sp<AMessage> extra;
968        if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
969            int64_t resumeAtMediaTimeUs;
970            if (extra->findInt64(
971                        "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
972                ALOGI("[%s] suppressing rendering until %lld us",
973                        mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
974                mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
975            }
976        }
977
978        int64_t timeUs = 0;
979        uint32_t flags = 0;
980        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
981
982        int32_t eos, csd;
983        // we do not expect SYNCFRAME for decoder
984        if (buffer->meta()->findInt32("eos", &eos) && eos) {
985            flags |= MediaCodec::BUFFER_FLAG_EOS;
986        } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
987            flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
988        }
989
990        // Modular DRM
991        MediaBuffer *mediaBuf = NULL;
992        NuPlayerDrm::CryptoInfo *cryptInfo = NULL;
993
994        // copy into codec buffer
995        if (needsCopy) {
996            if (buffer->size() > codecBuffer->capacity()) {
997                handleError(ERROR_BUFFER_TOO_SMALL);
998                mDequeuedInputBuffers.push_back(bufferIx);
999                return false;
1000            }
1001
1002            if (buffer->data() != NULL) {
1003                codecBuffer->setRange(0, buffer->size());
1004                memcpy(codecBuffer->data(), buffer->data(), buffer->size());
1005            } else { // No buffer->data()
1006                //Modular DRM
1007                mediaBuf = (MediaBuffer*)buffer->getMediaBufferBase();
1008                if (mediaBuf != NULL) {
1009                    codecBuffer->setRange(0, mediaBuf->size());
1010                    memcpy(codecBuffer->data(), mediaBuf->data(), mediaBuf->size());
1011
1012                    sp<MetaData> meta_data = mediaBuf->meta_data();
1013                    cryptInfo = NuPlayerDrm::getSampleCryptoInfo(meta_data);
1014
1015                    // since getMediaBuffer() has incremented the refCount
1016                    mediaBuf->release();
1017                } else { // No mediaBuf
1018                    ALOGE("onInputBufferFetched: buffer->data()/mediaBuf are NULL for %p",
1019                            buffer.get());
1020                    handleError(UNKNOWN_ERROR);
1021                    return false;
1022                }
1023            } // buffer->data()
1024        } // needsCopy
1025
1026        status_t err;
1027        AString errorDetailMsg;
1028        if (cryptInfo != NULL) {
1029            err = mCodec->queueSecureInputBuffer(
1030                    bufferIx,
1031                    codecBuffer->offset(),
1032                    cryptInfo->subSamples,
1033                    cryptInfo->numSubSamples,
1034                    cryptInfo->key,
1035                    cryptInfo->iv,
1036                    cryptInfo->mode,
1037                    cryptInfo->pattern,
1038                    timeUs,
1039                    flags,
1040                    &errorDetailMsg);
1041            // synchronous call so done with cryptInfo here
1042            free(cryptInfo);
1043        } else {
1044            err = mCodec->queueInputBuffer(
1045                    bufferIx,
1046                    codecBuffer->offset(),
1047                    codecBuffer->size(),
1048                    timeUs,
1049                    flags,
1050                    &errorDetailMsg);
1051        } // no cryptInfo
1052
1053        if (err != OK) {
1054            ALOGE("onInputBufferFetched: queue%sInputBuffer failed for %s (err=%d, %s)",
1055                    (cryptInfo != NULL ? "Secure" : ""),
1056                    mComponentName.c_str(), err, errorDetailMsg.c_str());
1057            handleError(err);
1058        } else {
1059            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
1060        }
1061
1062    }   // buffer != NULL
1063    return true;
1064}
1065
1066void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
1067    status_t err;
1068    int32_t render;
1069    size_t bufferIx;
1070    int32_t eos;
1071    CHECK(msg->findSize("buffer-ix", &bufferIx));
1072
1073    if (!mIsAudio) {
1074        int64_t timeUs;
1075        sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx];
1076        buffer->meta()->findInt64("timeUs", &timeUs);
1077
1078        if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
1079            mCCDecoder->display(timeUs);
1080        }
1081    }
1082
1083    if (msg->findInt32("render", &render) && render) {
1084        int64_t timestampNs;
1085        CHECK(msg->findInt64("timestampNs", &timestampNs));
1086        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
1087    } else {
1088        mNumOutputFramesDropped += !mIsAudio;
1089        err = mCodec->releaseOutputBuffer(bufferIx);
1090    }
1091    if (err != OK) {
1092        ALOGE("failed to release output buffer for %s (err=%d)",
1093                mComponentName.c_str(), err);
1094        handleError(err);
1095    }
1096    if (msg->findInt32("eos", &eos) && eos
1097            && isDiscontinuityPending()) {
1098        finishHandleDiscontinuity(true /* flushOnTimeChange */);
1099    }
1100}
1101
1102bool NuPlayer::Decoder::isDiscontinuityPending() const {
1103    return mFormatChangePending || mTimeChangePending;
1104}
1105
1106void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
1107    ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
1108            mFormatChangePending, mTimeChangePending, flushOnTimeChange);
1109
1110    // If we have format change, pause and wait to be killed;
1111    // If we have time change only, flush and restart fetching.
1112
1113    if (mFormatChangePending) {
1114        mPaused = true;
1115    } else if (mTimeChangePending) {
1116        if (flushOnTimeChange) {
1117            doFlush(false /* notifyComplete */);
1118            signalResume(false /* notifyComplete */);
1119        }
1120    }
1121
1122    // Notify NuPlayer to either shutdown decoder, or rescan sources
1123    sp<AMessage> msg = mNotify->dup();
1124    msg->setInt32("what", kWhatInputDiscontinuity);
1125    msg->setInt32("formatChange", mFormatChangePending);
1126    msg->post();
1127
1128    mFormatChangePending = false;
1129    mTimeChangePending = false;
1130}
1131
1132bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
1133        const sp<AMessage> &targetFormat) const {
1134    if (targetFormat == NULL) {
1135        return true;
1136    }
1137
1138    AString mime;
1139    if (!targetFormat->findString("mime", &mime)) {
1140        return false;
1141    }
1142
1143    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
1144        // field-by-field comparison
1145        const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
1146        for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
1147            int32_t oldVal, newVal;
1148            if (!mInputFormat->findInt32(keys[i], &oldVal) ||
1149                    !targetFormat->findInt32(keys[i], &newVal) ||
1150                    oldVal != newVal) {
1151                return false;
1152            }
1153        }
1154
1155        sp<ABuffer> oldBuf, newBuf;
1156        if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
1157                targetFormat->findBuffer("csd-0", &newBuf)) {
1158            if (oldBuf->size() != newBuf->size()) {
1159                return false;
1160            }
1161            return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
1162        }
1163    }
1164    return false;
1165}
1166
1167bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
1168    if (mInputFormat == NULL) {
1169        return false;
1170    }
1171
1172    if (targetFormat == NULL) {
1173        return true;
1174    }
1175
1176    AString oldMime, newMime;
1177    if (!mInputFormat->findString("mime", &oldMime)
1178            || !targetFormat->findString("mime", &newMime)
1179            || !(oldMime == newMime)) {
1180        return false;
1181    }
1182
1183    bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
1184    bool seamless;
1185    if (audio) {
1186        seamless = supportsSeamlessAudioFormatChange(targetFormat);
1187    } else {
1188        int32_t isAdaptive;
1189        seamless = (mCodec != NULL &&
1190                mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
1191                isAdaptive);
1192    }
1193
1194    ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
1195    return seamless;
1196}
1197
1198void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
1199    if (format == NULL) {
1200        return;
1201    }
1202    mCSDsForCurrentFormat.clear();
1203    for (int32_t i = 0; ; ++i) {
1204        AString tag = "csd-";
1205        tag.append(i);
1206        sp<ABuffer> buffer;
1207        if (!format->findBuffer(tag.c_str(), &buffer)) {
1208            break;
1209        }
1210        mCSDsForCurrentFormat.push(buffer);
1211    }
1212}
1213
1214void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
1215    if (mResumePending) {
1216        mResumePending = false;
1217
1218        sp<AMessage> notify = mNotify->dup();
1219        notify->setInt32("what", kWhatResumeCompleted);
1220        notify->post();
1221    }
1222}
1223
1224}  // namespace android
1225
1226