1/*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerDecoder"
19#include <utils/Log.h>
20#include <inttypes.h>
21
22#include <algorithm>
23
24#include "NuPlayerCCDecoder.h"
25#include "NuPlayerDecoder.h"
26#include "NuPlayerDrm.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29
30#include <cutils/properties.h>
31#include <media/ICrypto.h>
32#include <media/MediaBufferHolder.h>
33#include <media/MediaCodecBuffer.h>
34#include <media/stagefright/foundation/ABuffer.h>
35#include <media/stagefright/foundation/ADebug.h>
36#include <media/stagefright/foundation/AMessage.h>
37#include <media/stagefright/foundation/avc_utils.h>
38#include <media/stagefright/MediaBuffer.h>
39#include <media/stagefright/MediaCodec.h>
40#include <media/stagefright/MediaDefs.h>
41#include <media/stagefright/MediaErrors.h>
42#include <media/stagefright/SurfaceUtils.h>
43#include <gui/Surface.h>
44
45#include "ATSParser.h"
46
47namespace android {
48
49static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display
50
51// The default total video frame rate of a stream when that info is not available from
52// the source.
53static float kDefaultVideoFrameRateTotal = 30.f;
54
55static inline bool getAudioDeepBufferSetting() {
56    return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
57}
58
59NuPlayer::Decoder::Decoder(
60        const sp<AMessage> &notify,
61        const sp<Source> &source,
62        pid_t pid,
63        uid_t uid,
64        const sp<Renderer> &renderer,
65        const sp<Surface> &surface,
66        const sp<CCDecoder> &ccDecoder)
67    : DecoderBase(notify),
68      mSurface(surface),
69      mSource(source),
70      mRenderer(renderer),
71      mCCDecoder(ccDecoder),
72      mPid(pid),
73      mUid(uid),
74      mSkipRenderingUntilMediaTimeUs(-1ll),
75      mNumFramesTotal(0ll),
76      mNumInputFramesDropped(0ll),
77      mNumOutputFramesDropped(0ll),
78      mVideoWidth(0),
79      mVideoHeight(0),
80      mIsAudio(true),
81      mIsVideoAVC(false),
82      mIsSecure(false),
83      mIsEncrypted(false),
84      mIsEncryptedObservedEarlier(false),
85      mFormatChangePending(false),
86      mTimeChangePending(false),
87      mFrameRateTotal(kDefaultVideoFrameRateTotal),
88      mPlaybackSpeed(1.0f),
89      mNumVideoTemporalLayerTotal(1), // decode all layers
90      mNumVideoTemporalLayerAllowed(1),
91      mCurrentMaxVideoTemporalLayerId(0),
92      mResumePending(false),
93      mComponentName("decoder") {
94    mCodecLooper = new ALooper;
95    mCodecLooper->setName("NPDecoder-CL");
96    mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
97    mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;
98}
99
100NuPlayer::Decoder::~Decoder() {
101    // Need to stop looper first since mCodec could be accessed on the mDecoderLooper.
102    stopLooper();
103    if (mCodec != NULL) {
104        mCodec->release();
105    }
106    releaseAndResetMediaBuffers();
107}
108
109sp<AMessage> NuPlayer::Decoder::getStats() const {
110    mStats->setInt64("frames-total", mNumFramesTotal);
111    mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
112    mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
113    return mStats;
114}
115
116status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
117    if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
118        return BAD_VALUE;
119    }
120
121    sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
122
123    msg->setObject("surface", surface);
124    sp<AMessage> response;
125    status_t err = msg->postAndAwaitResponse(&response);
126    if (err == OK && response != NULL) {
127        CHECK(response->findInt32("err", &err));
128    }
129    return err;
130}
131
132void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
133    ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
134
135    switch (msg->what()) {
136        case kWhatCodecNotify:
137        {
138            int32_t cbID;
139            CHECK(msg->findInt32("callbackID", &cbID));
140
141            ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
142                    mIsAudio ? "audio" : "video", cbID, mPaused);
143
144            if (mPaused) {
145                break;
146            }
147
148            switch (cbID) {
149                case MediaCodec::CB_INPUT_AVAILABLE:
150                {
151                    int32_t index;
152                    CHECK(msg->findInt32("index", &index));
153
154                    handleAnInputBuffer(index);
155                    break;
156                }
157
158                case MediaCodec::CB_OUTPUT_AVAILABLE:
159                {
160                    int32_t index;
161                    size_t offset;
162                    size_t size;
163                    int64_t timeUs;
164                    int32_t flags;
165
166                    CHECK(msg->findInt32("index", &index));
167                    CHECK(msg->findSize("offset", &offset));
168                    CHECK(msg->findSize("size", &size));
169                    CHECK(msg->findInt64("timeUs", &timeUs));
170                    CHECK(msg->findInt32("flags", &flags));
171
172                    handleAnOutputBuffer(index, offset, size, timeUs, flags);
173                    break;
174                }
175
176                case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
177                {
178                    sp<AMessage> format;
179                    CHECK(msg->findMessage("format", &format));
180
181                    handleOutputFormatChange(format);
182                    break;
183                }
184
185                case MediaCodec::CB_ERROR:
186                {
187                    status_t err;
188                    CHECK(msg->findInt32("err", &err));
189                    ALOGE("Decoder (%s) reported error : 0x%x",
190                            mIsAudio ? "audio" : "video", err);
191
192                    handleError(err);
193                    break;
194                }
195
196                default:
197                {
198                    TRESPASS();
199                    break;
200                }
201            }
202
203            break;
204        }
205
206        case kWhatRenderBuffer:
207        {
208            if (!isStaleReply(msg)) {
209                onRenderBuffer(msg);
210            }
211            break;
212        }
213
214        case kWhatAudioOutputFormatChanged:
215        {
216            if (!isStaleReply(msg)) {
217                status_t err;
218                if (msg->findInt32("err", &err) && err != OK) {
219                    ALOGE("Renderer reported 0x%x when changing audio output format", err);
220                    handleError(err);
221                }
222            }
223            break;
224        }
225
226        case kWhatSetVideoSurface:
227        {
228            sp<AReplyToken> replyID;
229            CHECK(msg->senderAwaitsResponse(&replyID));
230
231            sp<RefBase> obj;
232            CHECK(msg->findObject("surface", &obj));
233            sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
234            int32_t err = INVALID_OPERATION;
235            // NOTE: in practice mSurface is always non-null, but checking here for completeness
236            if (mCodec != NULL && mSurface != NULL) {
237                // TODO: once AwesomePlayer is removed, remove this automatic connecting
238                // to the surface by MediaPlayerService.
239                //
240                // at this point MediaPlayerService::client has already connected to the
241                // surface, which MediaCodec does not expect
242                err = nativeWindowDisconnect(surface.get(), "kWhatSetVideoSurface(surface)");
243                if (err == OK) {
244                    err = mCodec->setSurface(surface);
245                    ALOGI_IF(err, "codec setSurface returned: %d", err);
246                    if (err == OK) {
247                        // reconnect to the old surface as MPS::Client will expect to
248                        // be able to disconnect from it.
249                        (void)nativeWindowConnect(mSurface.get(), "kWhatSetVideoSurface(mSurface)");
250                        mSurface = surface;
251                    }
252                }
253                if (err != OK) {
254                    // reconnect to the new surface on error as MPS::Client will expect to
255                    // be able to disconnect from it.
256                    (void)nativeWindowConnect(surface.get(), "kWhatSetVideoSurface(err)");
257                }
258            }
259
260            sp<AMessage> response = new AMessage;
261            response->setInt32("err", err);
262            response->postReply(replyID);
263            break;
264        }
265
266        case kWhatDrmReleaseCrypto:
267        {
268            ALOGV("kWhatDrmReleaseCrypto");
269            onReleaseCrypto(msg);
270            break;
271        }
272
273        default:
274            DecoderBase::onMessageReceived(msg);
275            break;
276    }
277}
278
279void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
280    CHECK(mCodec == NULL);
281
282    mFormatChangePending = false;
283    mTimeChangePending = false;
284
285    ++mBufferGeneration;
286
287    AString mime;
288    CHECK(format->findString("mime", &mime));
289
290    mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
291    mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
292
293    mComponentName = mime;
294    mComponentName.append(" decoder");
295    ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
296
297    mCodec = MediaCodec::CreateByType(
298            mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid);
299    int32_t secure = 0;
300    if (format->findInt32("secure", &secure) && secure != 0) {
301        if (mCodec != NULL) {
302            mCodec->getName(&mComponentName);
303            mComponentName.append(".secure");
304            mCodec->release();
305            ALOGI("[%s] creating", mComponentName.c_str());
306            mCodec = MediaCodec::CreateByComponentName(
307                    mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid);
308        }
309    }
310    if (mCodec == NULL) {
311        ALOGE("Failed to create %s%s decoder",
312                (secure ? "secure " : ""), mime.c_str());
313        handleError(UNKNOWN_ERROR);
314        return;
315    }
316    mIsSecure = secure;
317
318    mCodec->getName(&mComponentName);
319
320    status_t err;
321    if (mSurface != NULL) {
322        // disconnect from surface as MediaCodec will reconnect
323        err = nativeWindowDisconnect(mSurface.get(), "onConfigure");
324        // We treat this as a warning, as this is a preparatory step.
325        // Codec will try to connect to the surface, which is where
326        // any error signaling will occur.
327        ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
328    }
329
330    // Modular DRM
331    void *pCrypto;
332    if (!format->findPointer("crypto", &pCrypto)) {
333        pCrypto = NULL;
334    }
335    sp<ICrypto> crypto = (ICrypto*)pCrypto;
336    // non-encrypted source won't have a crypto
337    mIsEncrypted = (crypto != NULL);
338    // configure is called once; still using OR in case the behavior changes.
339    mIsEncryptedObservedEarlier = mIsEncryptedObservedEarlier || mIsEncrypted;
340    ALOGV("onConfigure mCrypto: %p (%d)  mIsSecure: %d",
341            crypto.get(), (crypto != NULL ? crypto->getStrongCount() : 0), mIsSecure);
342
343    err = mCodec->configure(
344            format, mSurface, crypto, 0 /* flags */);
345
346    if (err != OK) {
347        ALOGE("Failed to configure [%s] decoder (err=%d)", mComponentName.c_str(), err);
348        mCodec->release();
349        mCodec.clear();
350        handleError(err);
351        return;
352    }
353    rememberCodecSpecificData(format);
354
355    // the following should work in configured state
356    CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
357    CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
358
359    mStats->setString("mime", mime.c_str());
360    mStats->setString("component-name", mComponentName.c_str());
361
362    if (!mIsAudio) {
363        int32_t width, height;
364        if (mOutputFormat->findInt32("width", &width)
365                && mOutputFormat->findInt32("height", &height)) {
366            mStats->setInt32("width", width);
367            mStats->setInt32("height", height);
368        }
369    }
370
371    sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
372    mCodec->setCallback(reply);
373
374    err = mCodec->start();
375    if (err != OK) {
376        ALOGE("Failed to start [%s] decoder (err=%d)", mComponentName.c_str(), err);
377        mCodec->release();
378        mCodec.clear();
379        handleError(err);
380        return;
381    }
382
383    releaseAndResetMediaBuffers();
384
385    mPaused = false;
386    mResumePending = false;
387}
388
389void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) {
390    bool needAdjustLayers = false;
391    float frameRateTotal;
392    if (params->findFloat("frame-rate-total", &frameRateTotal)
393            && mFrameRateTotal != frameRateTotal) {
394        needAdjustLayers = true;
395        mFrameRateTotal = frameRateTotal;
396    }
397
398    int32_t numVideoTemporalLayerTotal;
399    if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal)
400            && numVideoTemporalLayerTotal >= 0
401            && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers
402            && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) {
403        needAdjustLayers = true;
404        mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1);
405    }
406
407    if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) {
408        // TODO: For now, layer fps is calculated for some specific architectures.
409        // But it really should be extracted from the stream.
410        mVideoTemporalLayerAggregateFps[0] =
411            mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1));
412        for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) {
413            mVideoTemporalLayerAggregateFps[i] =
414                mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i))
415                + mVideoTemporalLayerAggregateFps[i - 1];
416        }
417    }
418
419    float playbackSpeed;
420    if (params->findFloat("playback-speed", &playbackSpeed)
421            && mPlaybackSpeed != playbackSpeed) {
422        needAdjustLayers = true;
423        mPlaybackSpeed = playbackSpeed;
424    }
425
426    if (needAdjustLayers) {
427        float decodeFrameRate = mFrameRateTotal;
428        // enable temporal layering optimization only if we know the layering depth
429        if (mNumVideoTemporalLayerTotal > 1) {
430            int32_t layerId;
431            for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) {
432                if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed
433                        >= kDisplayRefreshingRate * 0.9) {
434                    break;
435                }
436            }
437            mNumVideoTemporalLayerAllowed = layerId + 1;
438            decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId];
439        }
440        ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g",
441                mNumVideoTemporalLayerAllowed, decodeFrameRate);
442
443        if (mCodec == NULL) {
444            ALOGW("onSetParameters called before codec is created.");
445            return;
446        }
447
448        sp<AMessage> codecParams = new AMessage();
449        codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed);
450        mCodec->setParameters(codecParams);
451    }
452}
453
454void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
455    mRenderer = renderer;
456}
457
458void NuPlayer::Decoder::onResume(bool notifyComplete) {
459    mPaused = false;
460
461    if (notifyComplete) {
462        mResumePending = true;
463    }
464
465    if (mCodec == NULL) {
466        ALOGE("[%s] onResume without a valid codec", mComponentName.c_str());
467        handleError(NO_INIT);
468        return;
469    }
470    mCodec->start();
471}
472
473void NuPlayer::Decoder::doFlush(bool notifyComplete) {
474    if (mCCDecoder != NULL) {
475        mCCDecoder->flush();
476    }
477
478    if (mRenderer != NULL) {
479        mRenderer->flush(mIsAudio, notifyComplete);
480        mRenderer->signalTimeDiscontinuity();
481    }
482
483    status_t err = OK;
484    if (mCodec != NULL) {
485        err = mCodec->flush();
486        mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
487        ++mBufferGeneration;
488    }
489
490    if (err != OK) {
491        ALOGE("failed to flush [%s] (err=%d)", mComponentName.c_str(), err);
492        handleError(err);
493        // finish with posting kWhatFlushCompleted.
494        // we attempt to release the buffers even if flush fails.
495    }
496    releaseAndResetMediaBuffers();
497    mPaused = true;
498}
499
500
501void NuPlayer::Decoder::onFlush() {
502    doFlush(true);
503
504    if (isDiscontinuityPending()) {
505        // This could happen if the client starts seeking/shutdown
506        // after we queued an EOS for discontinuities.
507        // We can consider discontinuity handled.
508        finishHandleDiscontinuity(false /* flushOnTimeChange */);
509    }
510
511    sp<AMessage> notify = mNotify->dup();
512    notify->setInt32("what", kWhatFlushCompleted);
513    notify->post();
514}
515
516void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
517    status_t err = OK;
518
519    // if there is a pending resume request, notify complete now
520    notifyResumeCompleteIfNecessary();
521
522    if (mCodec != NULL) {
523        err = mCodec->release();
524        mCodec = NULL;
525        ++mBufferGeneration;
526
527        if (mSurface != NULL) {
528            // reconnect to surface as MediaCodec disconnected from it
529            status_t error = nativeWindowConnect(mSurface.get(), "onShutdown");
530            ALOGW_IF(error != NO_ERROR,
531                    "[%s] failed to connect to native window, error=%d",
532                    mComponentName.c_str(), error);
533        }
534        mComponentName = "decoder";
535    }
536
537    releaseAndResetMediaBuffers();
538
539    if (err != OK) {
540        ALOGE("failed to release [%s] (err=%d)", mComponentName.c_str(), err);
541        handleError(err);
542        // finish with posting kWhatShutdownCompleted.
543    }
544
545    if (notifyComplete) {
546        sp<AMessage> notify = mNotify->dup();
547        notify->setInt32("what", kWhatShutdownCompleted);
548        notify->post();
549        mPaused = true;
550    }
551}
552
553/*
554 * returns true if we should request more data
555 */
556bool NuPlayer::Decoder::doRequestBuffers() {
557    if (isDiscontinuityPending()) {
558        return false;
559    }
560    status_t err = OK;
561    while (err == OK && !mDequeuedInputBuffers.empty()) {
562        size_t bufferIx = *mDequeuedInputBuffers.begin();
563        sp<AMessage> msg = new AMessage();
564        msg->setSize("buffer-ix", bufferIx);
565        err = fetchInputData(msg);
566        if (err != OK && err != ERROR_END_OF_STREAM) {
567            // if EOS, need to queue EOS buffer
568            break;
569        }
570        mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
571
572        if (!mPendingInputMessages.empty()
573                || !onInputBufferFetched(msg)) {
574            mPendingInputMessages.push_back(msg);
575        }
576    }
577
578    return err == -EWOULDBLOCK
579            && mSource->feedMoreTSData() == OK;
580}
581
582void NuPlayer::Decoder::handleError(int32_t err)
583{
584    // We cannot immediately release the codec due to buffers still outstanding
585    // in the renderer.  We signal to the player the error so it can shutdown/release the
586    // decoder after flushing and increment the generation to discard unnecessary messages.
587
588    ++mBufferGeneration;
589
590    sp<AMessage> notify = mNotify->dup();
591    notify->setInt32("what", kWhatError);
592    notify->setInt32("err", err);
593    notify->post();
594}
595
596status_t NuPlayer::Decoder::releaseCrypto()
597{
598    ALOGV("releaseCrypto");
599
600    sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this);
601
602    sp<AMessage> response;
603    status_t status = msg->postAndAwaitResponse(&response);
604    if (status == OK && response != NULL) {
605        CHECK(response->findInt32("status", &status));
606        ALOGV("releaseCrypto ret: %d ", status);
607    } else {
608        ALOGE("releaseCrypto err: %d", status);
609    }
610
611    return status;
612}
613
614void NuPlayer::Decoder::onReleaseCrypto(const sp<AMessage>& msg)
615{
616    status_t status = INVALID_OPERATION;
617    if (mCodec != NULL) {
618        status = mCodec->releaseCrypto();
619    } else {
620        // returning OK if the codec has been already released
621        status = OK;
622        ALOGE("onReleaseCrypto No mCodec. err: %d", status);
623    }
624
625    sp<AMessage> response = new AMessage;
626    response->setInt32("status", status);
627    // Clearing the state as it's tied to crypto. mIsEncryptedObservedEarlier is sticky though
628    // and lasts for the lifetime of this codec. See its use in fetchInputData.
629    mIsEncrypted = false;
630
631    sp<AReplyToken> replyID;
632    CHECK(msg->senderAwaitsResponse(&replyID));
633    response->postReply(replyID);
634}
635
636bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
637    if (isDiscontinuityPending()) {
638        return false;
639    }
640
641    if (mCodec == NULL) {
642        ALOGE("[%s] handleAnInputBuffer without a valid codec", mComponentName.c_str());
643        handleError(NO_INIT);
644        return false;
645    }
646
647    sp<MediaCodecBuffer> buffer;
648    mCodec->getInputBuffer(index, &buffer);
649
650    if (buffer == NULL) {
651        ALOGE("[%s] handleAnInputBuffer, failed to get input buffer", mComponentName.c_str());
652        handleError(UNKNOWN_ERROR);
653        return false;
654    }
655
656    if (index >= mInputBuffers.size()) {
657        for (size_t i = mInputBuffers.size(); i <= index; ++i) {
658            mInputBuffers.add();
659            mMediaBuffers.add();
660            mInputBufferIsDequeued.add();
661            mMediaBuffers.editItemAt(i) = NULL;
662            mInputBufferIsDequeued.editItemAt(i) = false;
663        }
664    }
665    mInputBuffers.editItemAt(index) = buffer;
666
667    //CHECK_LT(bufferIx, mInputBuffers.size());
668
669    if (mMediaBuffers[index] != NULL) {
670        mMediaBuffers[index]->release();
671        mMediaBuffers.editItemAt(index) = NULL;
672    }
673    mInputBufferIsDequeued.editItemAt(index) = true;
674
675    if (!mCSDsToSubmit.isEmpty()) {
676        sp<AMessage> msg = new AMessage();
677        msg->setSize("buffer-ix", index);
678
679        sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
680        ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
681        msg->setBuffer("buffer", buffer);
682        mCSDsToSubmit.removeAt(0);
683        if (!onInputBufferFetched(msg)) {
684            handleError(UNKNOWN_ERROR);
685            return false;
686        }
687        return true;
688    }
689
690    while (!mPendingInputMessages.empty()) {
691        sp<AMessage> msg = *mPendingInputMessages.begin();
692        if (!onInputBufferFetched(msg)) {
693            break;
694        }
695        mPendingInputMessages.erase(mPendingInputMessages.begin());
696    }
697
698    if (!mInputBufferIsDequeued.editItemAt(index)) {
699        return true;
700    }
701
702    mDequeuedInputBuffers.push_back(index);
703
704    onRequestInputBuffers();
705    return true;
706}
707
708bool NuPlayer::Decoder::handleAnOutputBuffer(
709        size_t index,
710        size_t offset,
711        size_t size,
712        int64_t timeUs,
713        int32_t flags) {
714    if (mCodec == NULL) {
715        ALOGE("[%s] handleAnOutputBuffer without a valid codec", mComponentName.c_str());
716        handleError(NO_INIT);
717        return false;
718    }
719
720//    CHECK_LT(bufferIx, mOutputBuffers.size());
721    sp<MediaCodecBuffer> buffer;
722    mCodec->getOutputBuffer(index, &buffer);
723
724    if (buffer == NULL) {
725        ALOGE("[%s] handleAnOutputBuffer, failed to get output buffer", mComponentName.c_str());
726        handleError(UNKNOWN_ERROR);
727        return false;
728    }
729
730    if (index >= mOutputBuffers.size()) {
731        for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
732            mOutputBuffers.add();
733        }
734    }
735
736    mOutputBuffers.editItemAt(index) = buffer;
737
738    buffer->setRange(offset, size);
739    buffer->meta()->clear();
740    buffer->meta()->setInt64("timeUs", timeUs);
741
742    bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
743    // we do not expect CODECCONFIG or SYNCFRAME for decoder
744
745    sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
746    reply->setSize("buffer-ix", index);
747    reply->setInt32("generation", mBufferGeneration);
748    reply->setSize("size", size);
749
750    if (eos) {
751        ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
752
753        buffer->meta()->setInt32("eos", true);
754        reply->setInt32("eos", true);
755    }
756
757    mNumFramesTotal += !mIsAudio;
758
759    if (mSkipRenderingUntilMediaTimeUs >= 0) {
760        if (timeUs < mSkipRenderingUntilMediaTimeUs) {
761            ALOGV("[%s] dropping buffer at time %lld as requested.",
762                     mComponentName.c_str(), (long long)timeUs);
763
764            reply->post();
765            if (eos) {
766                notifyResumeCompleteIfNecessary();
767                if (mRenderer != NULL && !isDiscontinuityPending()) {
768                    mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
769                }
770            }
771            return true;
772        }
773
774        mSkipRenderingUntilMediaTimeUs = -1;
775    }
776
777    // wait until 1st frame comes out to signal resume complete
778    notifyResumeCompleteIfNecessary();
779
780    if (mRenderer != NULL) {
781        // send the buffer to renderer.
782        mRenderer->queueBuffer(mIsAudio, buffer, reply);
783        if (eos && !isDiscontinuityPending()) {
784            mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
785        }
786    }
787
788    return true;
789}
790
791void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
792    if (!mIsAudio) {
793        int32_t width, height;
794        if (format->findInt32("width", &width)
795                && format->findInt32("height", &height)) {
796            mStats->setInt32("width", width);
797            mStats->setInt32("height", height);
798        }
799        sp<AMessage> notify = mNotify->dup();
800        notify->setInt32("what", kWhatVideoSizeChanged);
801        notify->setMessage("format", format);
802        notify->post();
803    } else if (mRenderer != NULL) {
804        uint32_t flags;
805        int64_t durationUs;
806        bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
807        if (getAudioDeepBufferSetting() // override regardless of source duration
808                || (mSource->getDuration(&durationUs) == OK
809                        && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
810            flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
811        } else {
812            flags = AUDIO_OUTPUT_FLAG_NONE;
813        }
814
815        sp<AMessage> reply = new AMessage(kWhatAudioOutputFormatChanged, this);
816        reply->setInt32("generation", mBufferGeneration);
817        mRenderer->changeAudioFormat(
818                format, false /* offloadOnly */, hasVideo,
819                flags, mSource->isStreaming(), reply);
820    }
821}
822
823void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
824    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
825        if (mMediaBuffers[i] != NULL) {
826            mMediaBuffers[i]->release();
827            mMediaBuffers.editItemAt(i) = NULL;
828        }
829    }
830    mMediaBuffers.resize(mInputBuffers.size());
831    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
832        mMediaBuffers.editItemAt(i) = NULL;
833    }
834    mInputBufferIsDequeued.clear();
835    mInputBufferIsDequeued.resize(mInputBuffers.size());
836    for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
837        mInputBufferIsDequeued.editItemAt(i) = false;
838    }
839
840    mPendingInputMessages.clear();
841    mDequeuedInputBuffers.clear();
842    mSkipRenderingUntilMediaTimeUs = -1;
843}
844
845void NuPlayer::Decoder::requestCodecNotification() {
846    if (mCodec != NULL) {
847        sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
848        reply->setInt32("generation", mBufferGeneration);
849        mCodec->requestActivityNotification(reply);
850    }
851}
852
853bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
854    int32_t generation;
855    CHECK(msg->findInt32("generation", &generation));
856    return generation != mBufferGeneration;
857}
858
859status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
860    sp<ABuffer> accessUnit;
861    bool dropAccessUnit = true;
862    do {
863        status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
864
865        if (err == -EWOULDBLOCK) {
866            return err;
867        } else if (err != OK) {
868            if (err == INFO_DISCONTINUITY) {
869                int32_t type;
870                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
871
872                bool formatChange =
873                    (mIsAudio &&
874                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
875                    || (!mIsAudio &&
876                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
877
878                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
879
880                ALOGI("%s discontinuity (format=%d, time=%d)",
881                        mIsAudio ? "audio" : "video", formatChange, timeChange);
882
883                bool seamlessFormatChange = false;
884                sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
885                if (formatChange) {
886                    seamlessFormatChange =
887                        supportsSeamlessFormatChange(newFormat);
888                    // treat seamless format change separately
889                    formatChange = !seamlessFormatChange;
890                }
891
892                // For format or time change, return EOS to queue EOS input,
893                // then wait for EOS on output.
894                if (formatChange /* not seamless */) {
895                    mFormatChangePending = true;
896                    err = ERROR_END_OF_STREAM;
897                } else if (timeChange) {
898                    rememberCodecSpecificData(newFormat);
899                    mTimeChangePending = true;
900                    err = ERROR_END_OF_STREAM;
901                } else if (seamlessFormatChange) {
902                    // reuse existing decoder and don't flush
903                    rememberCodecSpecificData(newFormat);
904                    continue;
905                } else {
906                    // This stream is unaffected by the discontinuity
907                    return -EWOULDBLOCK;
908                }
909            }
910
911            // reply should only be returned without a buffer set
912            // when there is an error (including EOS)
913            CHECK(err != OK);
914
915            reply->setInt32("err", err);
916            return ERROR_END_OF_STREAM;
917        }
918
919        dropAccessUnit = false;
920        if (!mIsAudio && !mIsEncrypted) {
921            // Extra safeguard if higher-level behavior changes. Otherwise, not required now.
922            // Preventing the buffer from being processed (and sent to codec) if this is a later
923            // round of playback but this time without prepareDrm. Or if there is a race between
924            // stop (which is not blocking) and releaseDrm allowing buffers being processed after
925            // Crypto has been released (GenericSource currently prevents this race though).
926            // Particularly doing this check before IsAVCReferenceFrame call to prevent parsing
927            // of encrypted data.
928            if (mIsEncryptedObservedEarlier) {
929                ALOGE("fetchInputData: mismatched mIsEncrypted/mIsEncryptedObservedEarlier (0/1)");
930
931                return INVALID_OPERATION;
932            }
933
934            int32_t layerId = 0;
935            bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId);
936            if (mRenderer->getVideoLateByUs() > 100000ll
937                    && mIsVideoAVC
938                    && !IsAVCReferenceFrame(accessUnit)) {
939                dropAccessUnit = true;
940            } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) {
941                // Add only one layer each time.
942                if (layerId > mCurrentMaxVideoTemporalLayerId + 1
943                        || layerId >= mNumVideoTemporalLayerAllowed) {
944                    dropAccessUnit = true;
945                    ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d",
946                            layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed,
947                            mCurrentMaxVideoTemporalLayerId);
948                } else if (layerId > mCurrentMaxVideoTemporalLayerId) {
949                    mCurrentMaxVideoTemporalLayerId = layerId;
950                } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1
951                        && IsIDR(accessUnit->data(), accessUnit->size())) {
952                    mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1;
953                }
954            }
955            if (dropAccessUnit) {
956                if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) {
957                    mCurrentMaxVideoTemporalLayerId = layerId - 1;
958                }
959                ++mNumInputFramesDropped;
960            }
961        }
962    } while (dropAccessUnit);
963
964    // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
965#if 0
966    int64_t mediaTimeUs;
967    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
968    ALOGV("[%s] feeding input buffer at media time %.3f",
969         mIsAudio ? "audio" : "video",
970         mediaTimeUs / 1E6);
971#endif
972
973    if (mCCDecoder != NULL) {
974        mCCDecoder->decode(accessUnit);
975    }
976
977    reply->setBuffer("buffer", accessUnit);
978
979    return OK;
980}
981
982bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
983    if (mCodec == NULL) {
984        ALOGE("[%s] onInputBufferFetched without a valid codec", mComponentName.c_str());
985        handleError(NO_INIT);
986        return false;
987    }
988
989    size_t bufferIx;
990    CHECK(msg->findSize("buffer-ix", &bufferIx));
991    CHECK_LT(bufferIx, mInputBuffers.size());
992    sp<MediaCodecBuffer> codecBuffer = mInputBuffers[bufferIx];
993
994    sp<ABuffer> buffer;
995    bool hasBuffer = msg->findBuffer("buffer", &buffer);
996    bool needsCopy = true;
997
998    if (buffer == NULL /* includes !hasBuffer */) {
999        int32_t streamErr = ERROR_END_OF_STREAM;
1000        CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
1001
1002        CHECK(streamErr != OK);
1003
1004        // attempt to queue EOS
1005        status_t err = mCodec->queueInputBuffer(
1006                bufferIx,
1007                0,
1008                0,
1009                0,
1010                MediaCodec::BUFFER_FLAG_EOS);
1011        if (err == OK) {
1012            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
1013        } else if (streamErr == ERROR_END_OF_STREAM) {
1014            streamErr = err;
1015            // err will not be ERROR_END_OF_STREAM
1016        }
1017
1018        if (streamErr != ERROR_END_OF_STREAM) {
1019            ALOGE("Stream error for [%s] (err=%d), EOS %s queued",
1020                    mComponentName.c_str(),
1021                    streamErr,
1022                    err == OK ? "successfully" : "unsuccessfully");
1023            handleError(streamErr);
1024        }
1025    } else {
1026        sp<AMessage> extra;
1027        if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
1028            int64_t resumeAtMediaTimeUs;
1029            if (extra->findInt64(
1030                        "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
1031                ALOGI("[%s] suppressing rendering until %lld us",
1032                        mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
1033                mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
1034            }
1035        }
1036
1037        int64_t timeUs = 0;
1038        uint32_t flags = 0;
1039        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1040
1041        int32_t eos, csd;
1042        // we do not expect SYNCFRAME for decoder
1043        if (buffer->meta()->findInt32("eos", &eos) && eos) {
1044            flags |= MediaCodec::BUFFER_FLAG_EOS;
1045        } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
1046            flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
1047        }
1048
1049        // Modular DRM
1050        MediaBufferBase *mediaBuf = NULL;
1051        NuPlayerDrm::CryptoInfo *cryptInfo = NULL;
1052
1053        // copy into codec buffer
1054        if (needsCopy) {
1055            if (buffer->size() > codecBuffer->capacity()) {
1056                handleError(ERROR_BUFFER_TOO_SMALL);
1057                mDequeuedInputBuffers.push_back(bufferIx);
1058                return false;
1059            }
1060
1061            if (buffer->data() != NULL) {
1062                codecBuffer->setRange(0, buffer->size());
1063                memcpy(codecBuffer->data(), buffer->data(), buffer->size());
1064            } else { // No buffer->data()
1065                //Modular DRM
1066                sp<RefBase> holder;
1067                if (buffer->meta()->findObject("mediaBufferHolder", &holder)) {
1068                    mediaBuf = (holder != nullptr) ?
1069                        static_cast<MediaBufferHolder*>(holder.get())->mediaBuffer() : nullptr;
1070                }
1071                if (mediaBuf != NULL) {
1072                    codecBuffer->setRange(0, mediaBuf->size());
1073                    memcpy(codecBuffer->data(), mediaBuf->data(), mediaBuf->size());
1074
1075                    MetaDataBase &meta_data = mediaBuf->meta_data();
1076                    cryptInfo = NuPlayerDrm::getSampleCryptoInfo(meta_data);
1077                } else { // No mediaBuf
1078                    ALOGE("onInputBufferFetched: buffer->data()/mediaBuf are NULL for %p",
1079                            buffer.get());
1080                    handleError(UNKNOWN_ERROR);
1081                    return false;
1082                }
1083            } // buffer->data()
1084        } // needsCopy
1085
1086        status_t err;
1087        AString errorDetailMsg;
1088        if (cryptInfo != NULL) {
1089            err = mCodec->queueSecureInputBuffer(
1090                    bufferIx,
1091                    codecBuffer->offset(),
1092                    cryptInfo->subSamples,
1093                    cryptInfo->numSubSamples,
1094                    cryptInfo->key,
1095                    cryptInfo->iv,
1096                    cryptInfo->mode,
1097                    cryptInfo->pattern,
1098                    timeUs,
1099                    flags,
1100                    &errorDetailMsg);
1101            // synchronous call so done with cryptInfo here
1102            free(cryptInfo);
1103        } else {
1104            err = mCodec->queueInputBuffer(
1105                    bufferIx,
1106                    codecBuffer->offset(),
1107                    codecBuffer->size(),
1108                    timeUs,
1109                    flags,
1110                    &errorDetailMsg);
1111        } // no cryptInfo
1112
1113        if (err != OK) {
1114            ALOGE("onInputBufferFetched: queue%sInputBuffer failed for [%s] (err=%d, %s)",
1115                    (cryptInfo != NULL ? "Secure" : ""),
1116                    mComponentName.c_str(), err, errorDetailMsg.c_str());
1117            handleError(err);
1118        } else {
1119            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
1120        }
1121
1122    }   // buffer != NULL
1123    return true;
1124}
1125
1126void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
1127    status_t err;
1128    int32_t render;
1129    size_t bufferIx;
1130    int32_t eos;
1131    size_t size;
1132    CHECK(msg->findSize("buffer-ix", &bufferIx));
1133
1134    if (!mIsAudio) {
1135        int64_t timeUs;
1136        sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx];
1137        buffer->meta()->findInt64("timeUs", &timeUs);
1138
1139        if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
1140            mCCDecoder->display(timeUs);
1141        }
1142    }
1143
1144    if (mCodec == NULL) {
1145        err = NO_INIT;
1146    } else if (msg->findInt32("render", &render) && render) {
1147        int64_t timestampNs;
1148        CHECK(msg->findInt64("timestampNs", &timestampNs));
1149        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
1150    } else {
1151        if (!msg->findInt32("eos", &eos) || !eos ||
1152                !msg->findSize("size", &size) || size) {
1153            mNumOutputFramesDropped += !mIsAudio;
1154        }
1155        err = mCodec->releaseOutputBuffer(bufferIx);
1156    }
1157    if (err != OK) {
1158        ALOGE("failed to release output buffer for [%s] (err=%d)",
1159                mComponentName.c_str(), err);
1160        handleError(err);
1161    }
1162    if (msg->findInt32("eos", &eos) && eos
1163            && isDiscontinuityPending()) {
1164        finishHandleDiscontinuity(true /* flushOnTimeChange */);
1165    }
1166}
1167
1168bool NuPlayer::Decoder::isDiscontinuityPending() const {
1169    return mFormatChangePending || mTimeChangePending;
1170}
1171
1172void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
1173    ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
1174            mFormatChangePending, mTimeChangePending, flushOnTimeChange);
1175
1176    // If we have format change, pause and wait to be killed;
1177    // If we have time change only, flush and restart fetching.
1178
1179    if (mFormatChangePending) {
1180        mPaused = true;
1181    } else if (mTimeChangePending) {
1182        if (flushOnTimeChange) {
1183            doFlush(false /* notifyComplete */);
1184            signalResume(false /* notifyComplete */);
1185        }
1186    }
1187
1188    // Notify NuPlayer to either shutdown decoder, or rescan sources
1189    sp<AMessage> msg = mNotify->dup();
1190    msg->setInt32("what", kWhatInputDiscontinuity);
1191    msg->setInt32("formatChange", mFormatChangePending);
1192    msg->post();
1193
1194    mFormatChangePending = false;
1195    mTimeChangePending = false;
1196}
1197
1198bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
1199        const sp<AMessage> &targetFormat) const {
1200    if (targetFormat == NULL) {
1201        return true;
1202    }
1203
1204    AString mime;
1205    if (!targetFormat->findString("mime", &mime)) {
1206        return false;
1207    }
1208
1209    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
1210        // field-by-field comparison
1211        const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
1212        for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
1213            int32_t oldVal, newVal;
1214            if (!mInputFormat->findInt32(keys[i], &oldVal) ||
1215                    !targetFormat->findInt32(keys[i], &newVal) ||
1216                    oldVal != newVal) {
1217                return false;
1218            }
1219        }
1220
1221        sp<ABuffer> oldBuf, newBuf;
1222        if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
1223                targetFormat->findBuffer("csd-0", &newBuf)) {
1224            if (oldBuf->size() != newBuf->size()) {
1225                return false;
1226            }
1227            return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
1228        }
1229    }
1230    return false;
1231}
1232
1233bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
1234    if (mInputFormat == NULL) {
1235        return false;
1236    }
1237
1238    if (targetFormat == NULL) {
1239        return true;
1240    }
1241
1242    AString oldMime, newMime;
1243    if (!mInputFormat->findString("mime", &oldMime)
1244            || !targetFormat->findString("mime", &newMime)
1245            || !(oldMime == newMime)) {
1246        return false;
1247    }
1248
1249    bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
1250    bool seamless;
1251    if (audio) {
1252        seamless = supportsSeamlessAudioFormatChange(targetFormat);
1253    } else {
1254        int32_t isAdaptive;
1255        seamless = (mCodec != NULL &&
1256                mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
1257                isAdaptive);
1258    }
1259
1260    ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
1261    return seamless;
1262}
1263
1264void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
1265    if (format == NULL) {
1266        return;
1267    }
1268    mCSDsForCurrentFormat.clear();
1269    for (int32_t i = 0; ; ++i) {
1270        AString tag = "csd-";
1271        tag.append(i);
1272        sp<ABuffer> buffer;
1273        if (!format->findBuffer(tag.c_str(), &buffer)) {
1274            break;
1275        }
1276        mCSDsForCurrentFormat.push(buffer);
1277    }
1278}
1279
1280void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
1281    if (mResumePending) {
1282        mResumePending = false;
1283
1284        sp<AMessage> notify = mNotify->dup();
1285        notify->setInt32("what", kWhatResumeCompleted);
1286        notify->post();
1287    }
1288}
1289
1290}  // namespace android
1291
1292