1/*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerDecoder"
19#include <utils/Log.h>
20#include <inttypes.h>
21
22#include <algorithm>
23
24#include "NuPlayerCCDecoder.h"
25#include "NuPlayerDecoder.h"
26#include "NuPlayerDrm.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29
30#include <cutils/properties.h>
31#include <media/ICrypto.h>
32#include <media/MediaCodecBuffer.h>
33#include <media/stagefright/foundation/ABuffer.h>
34#include <media/stagefright/foundation/ADebug.h>
35#include <media/stagefright/foundation/AMessage.h>
36#include <media/stagefright/MediaBuffer.h>
37#include <media/stagefright/MediaCodec.h>
38#include <media/stagefright/MediaDefs.h>
39#include <media/stagefright/MediaErrors.h>
40#include <media/stagefright/SurfaceUtils.h>
41#include <gui/Surface.h>
42
43#include "avc_utils.h"
44#include "ATSParser.h"
45
46namespace android {
47
48static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display
49
50// The default total video frame rate of a stream when that info is not available from
51// the source.
52static float kDefaultVideoFrameRateTotal = 30.f;
53
54static inline bool getAudioDeepBufferSetting() {
55    return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
56}
57
58NuPlayer::Decoder::Decoder(
59        const sp<AMessage> &notify,
60        const sp<Source> &source,
61        pid_t pid,
62        uid_t uid,
63        const sp<Renderer> &renderer,
64        const sp<Surface> &surface,
65        const sp<CCDecoder> &ccDecoder)
66    : DecoderBase(notify),
67      mSurface(surface),
68      mSource(source),
69      mRenderer(renderer),
70      mCCDecoder(ccDecoder),
71      mPid(pid),
72      mUid(uid),
73      mSkipRenderingUntilMediaTimeUs(-1ll),
74      mNumFramesTotal(0ll),
75      mNumInputFramesDropped(0ll),
76      mNumOutputFramesDropped(0ll),
77      mVideoWidth(0),
78      mVideoHeight(0),
79      mIsAudio(true),
80      mIsVideoAVC(false),
81      mIsSecure(false),
82      mIsEncrypted(false),
83      mIsEncryptedObservedEarlier(false),
84      mFormatChangePending(false),
85      mTimeChangePending(false),
86      mFrameRateTotal(kDefaultVideoFrameRateTotal),
87      mPlaybackSpeed(1.0f),
88      mNumVideoTemporalLayerTotal(1), // decode all layers
89      mNumVideoTemporalLayerAllowed(1),
90      mCurrentMaxVideoTemporalLayerId(0),
91      mResumePending(false),
92      mComponentName("decoder") {
93    mCodecLooper = new ALooper;
94    mCodecLooper->setName("NPDecoder-CL");
95    mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
96    mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;
97}
98
99NuPlayer::Decoder::~Decoder() {
100    // Need to stop looper first since mCodec could be accessed on the mDecoderLooper.
101    stopLooper();
102    if (mCodec != NULL) {
103        mCodec->release();
104    }
105    releaseAndResetMediaBuffers();
106}
107
108sp<AMessage> NuPlayer::Decoder::getStats() const {
109    mStats->setInt64("frames-total", mNumFramesTotal);
110    mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
111    mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
112    return mStats;
113}
114
115status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
116    if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
117        return BAD_VALUE;
118    }
119
120    sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
121
122    msg->setObject("surface", surface);
123    sp<AMessage> response;
124    status_t err = msg->postAndAwaitResponse(&response);
125    if (err == OK && response != NULL) {
126        CHECK(response->findInt32("err", &err));
127    }
128    return err;
129}
130
131void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
132    ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
133
134    switch (msg->what()) {
135        case kWhatCodecNotify:
136        {
137            int32_t cbID;
138            CHECK(msg->findInt32("callbackID", &cbID));
139
140            ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
141                    mIsAudio ? "audio" : "video", cbID, mPaused);
142
143            if (mPaused) {
144                break;
145            }
146
147            switch (cbID) {
148                case MediaCodec::CB_INPUT_AVAILABLE:
149                {
150                    int32_t index;
151                    CHECK(msg->findInt32("index", &index));
152
153                    handleAnInputBuffer(index);
154                    break;
155                }
156
157                case MediaCodec::CB_OUTPUT_AVAILABLE:
158                {
159                    int32_t index;
160                    size_t offset;
161                    size_t size;
162                    int64_t timeUs;
163                    int32_t flags;
164
165                    CHECK(msg->findInt32("index", &index));
166                    CHECK(msg->findSize("offset", &offset));
167                    CHECK(msg->findSize("size", &size));
168                    CHECK(msg->findInt64("timeUs", &timeUs));
169                    CHECK(msg->findInt32("flags", &flags));
170
171                    handleAnOutputBuffer(index, offset, size, timeUs, flags);
172                    break;
173                }
174
175                case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
176                {
177                    sp<AMessage> format;
178                    CHECK(msg->findMessage("format", &format));
179
180                    handleOutputFormatChange(format);
181                    break;
182                }
183
184                case MediaCodec::CB_ERROR:
185                {
186                    status_t err;
187                    CHECK(msg->findInt32("err", &err));
188                    ALOGE("Decoder (%s) reported error : 0x%x",
189                            mIsAudio ? "audio" : "video", err);
190
191                    handleError(err);
192                    break;
193                }
194
195                default:
196                {
197                    TRESPASS();
198                    break;
199                }
200            }
201
202            break;
203        }
204
205        case kWhatRenderBuffer:
206        {
207            if (!isStaleReply(msg)) {
208                onRenderBuffer(msg);
209            }
210            break;
211        }
212
213        case kWhatAudioOutputFormatChanged:
214        {
215            if (!isStaleReply(msg)) {
216                status_t err;
217                if (msg->findInt32("err", &err) && err != OK) {
218                    ALOGE("Renderer reported 0x%x when changing audio output format", err);
219                    handleError(err);
220                }
221            }
222            break;
223        }
224
225        case kWhatSetVideoSurface:
226        {
227            sp<AReplyToken> replyID;
228            CHECK(msg->senderAwaitsResponse(&replyID));
229
230            sp<RefBase> obj;
231            CHECK(msg->findObject("surface", &obj));
232            sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
233            int32_t err = INVALID_OPERATION;
234            // NOTE: in practice mSurface is always non-null, but checking here for completeness
235            if (mCodec != NULL && mSurface != NULL) {
236                // TODO: once AwesomePlayer is removed, remove this automatic connecting
237                // to the surface by MediaPlayerService.
238                //
239                // at this point MediaPlayerService::client has already connected to the
240                // surface, which MediaCodec does not expect
241                err = nativeWindowDisconnect(surface.get(), "kWhatSetVideoSurface(surface)");
242                if (err == OK) {
243                    err = mCodec->setSurface(surface);
244                    ALOGI_IF(err, "codec setSurface returned: %d", err);
245                    if (err == OK) {
246                        // reconnect to the old surface as MPS::Client will expect to
247                        // be able to disconnect from it.
248                        (void)nativeWindowConnect(mSurface.get(), "kWhatSetVideoSurface(mSurface)");
249                        mSurface = surface;
250                    }
251                }
252                if (err != OK) {
253                    // reconnect to the new surface on error as MPS::Client will expect to
254                    // be able to disconnect from it.
255                    (void)nativeWindowConnect(surface.get(), "kWhatSetVideoSurface(err)");
256                }
257            }
258
259            sp<AMessage> response = new AMessage;
260            response->setInt32("err", err);
261            response->postReply(replyID);
262            break;
263        }
264
265        case kWhatDrmReleaseCrypto:
266        {
267            ALOGV("kWhatDrmReleaseCrypto");
268            onReleaseCrypto(msg);
269            break;
270        }
271
272        default:
273            DecoderBase::onMessageReceived(msg);
274            break;
275    }
276}
277
278void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
279    CHECK(mCodec == NULL);
280
281    mFormatChangePending = false;
282    mTimeChangePending = false;
283
284    ++mBufferGeneration;
285
286    AString mime;
287    CHECK(format->findString("mime", &mime));
288
289    mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
290    mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
291
292    mComponentName = mime;
293    mComponentName.append(" decoder");
294    ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
295
296    mCodec = MediaCodec::CreateByType(
297            mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid);
298    int32_t secure = 0;
299    if (format->findInt32("secure", &secure) && secure != 0) {
300        if (mCodec != NULL) {
301            mCodec->getName(&mComponentName);
302            mComponentName.append(".secure");
303            mCodec->release();
304            ALOGI("[%s] creating", mComponentName.c_str());
305            mCodec = MediaCodec::CreateByComponentName(
306                    mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid);
307        }
308    }
309    if (mCodec == NULL) {
310        ALOGE("Failed to create %s%s decoder",
311                (secure ? "secure " : ""), mime.c_str());
312        handleError(UNKNOWN_ERROR);
313        return;
314    }
315    mIsSecure = secure;
316
317    mCodec->getName(&mComponentName);
318
319    status_t err;
320    if (mSurface != NULL) {
321        // disconnect from surface as MediaCodec will reconnect
322        err = nativeWindowDisconnect(mSurface.get(), "onConfigure");
323        // We treat this as a warning, as this is a preparatory step.
324        // Codec will try to connect to the surface, which is where
325        // any error signaling will occur.
326        ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
327    }
328
329    // Modular DRM
330    void *pCrypto;
331    if (!format->findPointer("crypto", &pCrypto)) {
332        pCrypto = NULL;
333    }
334    sp<ICrypto> crypto = (ICrypto*)pCrypto;
335    // non-encrypted source won't have a crypto
336    mIsEncrypted = (crypto != NULL);
337    // configure is called once; still using OR in case the behavior changes.
338    mIsEncryptedObservedEarlier = mIsEncryptedObservedEarlier || mIsEncrypted;
339    ALOGV("onConfigure mCrypto: %p (%d)  mIsSecure: %d",
340            crypto.get(), (crypto != NULL ? crypto->getStrongCount() : 0), mIsSecure);
341
342    err = mCodec->configure(
343            format, mSurface, crypto, 0 /* flags */);
344
345    if (err != OK) {
346        ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
347        mCodec->release();
348        mCodec.clear();
349        handleError(err);
350        return;
351    }
352    rememberCodecSpecificData(format);
353
354    // the following should work in configured state
355    CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
356    CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
357
358    mStats->setString("mime", mime.c_str());
359    mStats->setString("component-name", mComponentName.c_str());
360
361    if (!mIsAudio) {
362        int32_t width, height;
363        if (mOutputFormat->findInt32("width", &width)
364                && mOutputFormat->findInt32("height", &height)) {
365            mStats->setInt32("width", width);
366            mStats->setInt32("height", height);
367        }
368    }
369
370    sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
371    mCodec->setCallback(reply);
372
373    err = mCodec->start();
374    if (err != OK) {
375        ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
376        mCodec->release();
377        mCodec.clear();
378        handleError(err);
379        return;
380    }
381
382    releaseAndResetMediaBuffers();
383
384    mPaused = false;
385    mResumePending = false;
386}
387
388void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) {
389    bool needAdjustLayers = false;
390    float frameRateTotal;
391    if (params->findFloat("frame-rate-total", &frameRateTotal)
392            && mFrameRateTotal != frameRateTotal) {
393        needAdjustLayers = true;
394        mFrameRateTotal = frameRateTotal;
395    }
396
397    int32_t numVideoTemporalLayerTotal;
398    if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal)
399            && numVideoTemporalLayerTotal >= 0
400            && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers
401            && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) {
402        needAdjustLayers = true;
403        mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1);
404    }
405
406    if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) {
407        // TODO: For now, layer fps is calculated for some specific architectures.
408        // But it really should be extracted from the stream.
409        mVideoTemporalLayerAggregateFps[0] =
410            mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1));
411        for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) {
412            mVideoTemporalLayerAggregateFps[i] =
413                mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i))
414                + mVideoTemporalLayerAggregateFps[i - 1];
415        }
416    }
417
418    float playbackSpeed;
419    if (params->findFloat("playback-speed", &playbackSpeed)
420            && mPlaybackSpeed != playbackSpeed) {
421        needAdjustLayers = true;
422        mPlaybackSpeed = playbackSpeed;
423    }
424
425    if (needAdjustLayers) {
426        float decodeFrameRate = mFrameRateTotal;
427        // enable temporal layering optimization only if we know the layering depth
428        if (mNumVideoTemporalLayerTotal > 1) {
429            int32_t layerId;
430            for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) {
431                if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed
432                        >= kDisplayRefreshingRate * 0.9) {
433                    break;
434                }
435            }
436            mNumVideoTemporalLayerAllowed = layerId + 1;
437            decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId];
438        }
439        ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g",
440                mNumVideoTemporalLayerAllowed, decodeFrameRate);
441
442        if (mCodec == NULL) {
443            ALOGW("onSetParameters called before codec is created.");
444            return;
445        }
446
447        sp<AMessage> codecParams = new AMessage();
448        codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed);
449        mCodec->setParameters(codecParams);
450    }
451}
452
453void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
454    mRenderer = renderer;
455}
456
457void NuPlayer::Decoder::onResume(bool notifyComplete) {
458    mPaused = false;
459
460    if (notifyComplete) {
461        mResumePending = true;
462    }
463    mCodec->start();
464}
465
466void NuPlayer::Decoder::doFlush(bool notifyComplete) {
467    if (mCCDecoder != NULL) {
468        mCCDecoder->flush();
469    }
470
471    if (mRenderer != NULL) {
472        mRenderer->flush(mIsAudio, notifyComplete);
473        mRenderer->signalTimeDiscontinuity();
474    }
475
476    status_t err = OK;
477    if (mCodec != NULL) {
478        err = mCodec->flush();
479        mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
480        ++mBufferGeneration;
481    }
482
483    if (err != OK) {
484        ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
485        handleError(err);
486        // finish with posting kWhatFlushCompleted.
487        // we attempt to release the buffers even if flush fails.
488    }
489    releaseAndResetMediaBuffers();
490    mPaused = true;
491}
492
493
494void NuPlayer::Decoder::onFlush() {
495    doFlush(true);
496
497    if (isDiscontinuityPending()) {
498        // This could happen if the client starts seeking/shutdown
499        // after we queued an EOS for discontinuities.
500        // We can consider discontinuity handled.
501        finishHandleDiscontinuity(false /* flushOnTimeChange */);
502    }
503
504    sp<AMessage> notify = mNotify->dup();
505    notify->setInt32("what", kWhatFlushCompleted);
506    notify->post();
507}
508
509void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
510    status_t err = OK;
511
512    // if there is a pending resume request, notify complete now
513    notifyResumeCompleteIfNecessary();
514
515    if (mCodec != NULL) {
516        err = mCodec->release();
517        mCodec = NULL;
518        ++mBufferGeneration;
519
520        if (mSurface != NULL) {
521            // reconnect to surface as MediaCodec disconnected from it
522            status_t error = nativeWindowConnect(mSurface.get(), "onShutdown");
523            ALOGW_IF(error != NO_ERROR,
524                    "[%s] failed to connect to native window, error=%d",
525                    mComponentName.c_str(), error);
526        }
527        mComponentName = "decoder";
528    }
529
530    releaseAndResetMediaBuffers();
531
532    if (err != OK) {
533        ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
534        handleError(err);
535        // finish with posting kWhatShutdownCompleted.
536    }
537
538    if (notifyComplete) {
539        sp<AMessage> notify = mNotify->dup();
540        notify->setInt32("what", kWhatShutdownCompleted);
541        notify->post();
542        mPaused = true;
543    }
544}
545
546/*
547 * returns true if we should request more data
548 */
549bool NuPlayer::Decoder::doRequestBuffers() {
550    if (isDiscontinuityPending()) {
551        return false;
552    }
553    status_t err = OK;
554    while (err == OK && !mDequeuedInputBuffers.empty()) {
555        size_t bufferIx = *mDequeuedInputBuffers.begin();
556        sp<AMessage> msg = new AMessage();
557        msg->setSize("buffer-ix", bufferIx);
558        err = fetchInputData(msg);
559        if (err != OK && err != ERROR_END_OF_STREAM) {
560            // if EOS, need to queue EOS buffer
561            break;
562        }
563        mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
564
565        if (!mPendingInputMessages.empty()
566                || !onInputBufferFetched(msg)) {
567            mPendingInputMessages.push_back(msg);
568        }
569    }
570
571    return err == -EWOULDBLOCK
572            && mSource->feedMoreTSData() == OK;
573}
574
575void NuPlayer::Decoder::handleError(int32_t err)
576{
577    // We cannot immediately release the codec due to buffers still outstanding
578    // in the renderer.  We signal to the player the error so it can shutdown/release the
579    // decoder after flushing and increment the generation to discard unnecessary messages.
580
581    ++mBufferGeneration;
582
583    sp<AMessage> notify = mNotify->dup();
584    notify->setInt32("what", kWhatError);
585    notify->setInt32("err", err);
586    notify->post();
587}
588
589status_t NuPlayer::Decoder::releaseCrypto()
590{
591    ALOGV("releaseCrypto");
592
593    sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this);
594
595    sp<AMessage> response;
596    status_t status = msg->postAndAwaitResponse(&response);
597    if (status == OK && response != NULL) {
598        CHECK(response->findInt32("status", &status));
599        ALOGV("releaseCrypto ret: %d ", status);
600    } else {
601        ALOGE("releaseCrypto err: %d", status);
602    }
603
604    return status;
605}
606
607void NuPlayer::Decoder::onReleaseCrypto(const sp<AMessage>& msg)
608{
609    status_t status = INVALID_OPERATION;
610    if (mCodec != NULL) {
611        status = mCodec->releaseCrypto();
612    } else {
613        // returning OK if the codec has been already released
614        status = OK;
615        ALOGE("onReleaseCrypto No mCodec. err: %d", status);
616    }
617
618    sp<AMessage> response = new AMessage;
619    response->setInt32("status", status);
620    // Clearing the state as it's tied to crypto. mIsEncryptedObservedEarlier is sticky though
621    // and lasts for the lifetime of this codec. See its use in fetchInputData.
622    mIsEncrypted = false;
623
624    sp<AReplyToken> replyID;
625    CHECK(msg->senderAwaitsResponse(&replyID));
626    response->postReply(replyID);
627}
628
629bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
630    if (isDiscontinuityPending()) {
631        return false;
632    }
633
634    sp<MediaCodecBuffer> buffer;
635    mCodec->getInputBuffer(index, &buffer);
636
637    if (buffer == NULL) {
638        handleError(UNKNOWN_ERROR);
639        return false;
640    }
641
642    if (index >= mInputBuffers.size()) {
643        for (size_t i = mInputBuffers.size(); i <= index; ++i) {
644            mInputBuffers.add();
645            mMediaBuffers.add();
646            mInputBufferIsDequeued.add();
647            mMediaBuffers.editItemAt(i) = NULL;
648            mInputBufferIsDequeued.editItemAt(i) = false;
649        }
650    }
651    mInputBuffers.editItemAt(index) = buffer;
652
653    //CHECK_LT(bufferIx, mInputBuffers.size());
654
655    if (mMediaBuffers[index] != NULL) {
656        mMediaBuffers[index]->release();
657        mMediaBuffers.editItemAt(index) = NULL;
658    }
659    mInputBufferIsDequeued.editItemAt(index) = true;
660
661    if (!mCSDsToSubmit.isEmpty()) {
662        sp<AMessage> msg = new AMessage();
663        msg->setSize("buffer-ix", index);
664
665        sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
666        ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
667        msg->setBuffer("buffer", buffer);
668        mCSDsToSubmit.removeAt(0);
669        if (!onInputBufferFetched(msg)) {
670            handleError(UNKNOWN_ERROR);
671            return false;
672        }
673        return true;
674    }
675
676    while (!mPendingInputMessages.empty()) {
677        sp<AMessage> msg = *mPendingInputMessages.begin();
678        if (!onInputBufferFetched(msg)) {
679            break;
680        }
681        mPendingInputMessages.erase(mPendingInputMessages.begin());
682    }
683
684    if (!mInputBufferIsDequeued.editItemAt(index)) {
685        return true;
686    }
687
688    mDequeuedInputBuffers.push_back(index);
689
690    onRequestInputBuffers();
691    return true;
692}
693
694bool NuPlayer::Decoder::handleAnOutputBuffer(
695        size_t index,
696        size_t offset,
697        size_t size,
698        int64_t timeUs,
699        int32_t flags) {
700//    CHECK_LT(bufferIx, mOutputBuffers.size());
701    sp<MediaCodecBuffer> buffer;
702    mCodec->getOutputBuffer(index, &buffer);
703
704    if (buffer == NULL) {
705        handleError(UNKNOWN_ERROR);
706        return false;
707    }
708
709    if (index >= mOutputBuffers.size()) {
710        for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
711            mOutputBuffers.add();
712        }
713    }
714
715    mOutputBuffers.editItemAt(index) = buffer;
716
717    buffer->setRange(offset, size);
718    buffer->meta()->clear();
719    buffer->meta()->setInt64("timeUs", timeUs);
720
721    bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
722    // we do not expect CODECCONFIG or SYNCFRAME for decoder
723
724    sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
725    reply->setSize("buffer-ix", index);
726    reply->setInt32("generation", mBufferGeneration);
727
728    if (eos) {
729        ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
730
731        buffer->meta()->setInt32("eos", true);
732        reply->setInt32("eos", true);
733    } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
734        if (timeUs < mSkipRenderingUntilMediaTimeUs) {
735            ALOGV("[%s] dropping buffer at time %lld as requested.",
736                     mComponentName.c_str(), (long long)timeUs);
737
738            reply->post();
739            return true;
740        }
741
742        mSkipRenderingUntilMediaTimeUs = -1;
743    }
744
745    mNumFramesTotal += !mIsAudio;
746
747    // wait until 1st frame comes out to signal resume complete
748    notifyResumeCompleteIfNecessary();
749
750    if (mRenderer != NULL) {
751        // send the buffer to renderer.
752        mRenderer->queueBuffer(mIsAudio, buffer, reply);
753        if (eos && !isDiscontinuityPending()) {
754            mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
755        }
756    }
757
758    return true;
759}
760
761void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
762    if (!mIsAudio) {
763        int32_t width, height;
764        if (format->findInt32("width", &width)
765                && format->findInt32("height", &height)) {
766            mStats->setInt32("width", width);
767            mStats->setInt32("height", height);
768        }
769        sp<AMessage> notify = mNotify->dup();
770        notify->setInt32("what", kWhatVideoSizeChanged);
771        notify->setMessage("format", format);
772        notify->post();
773    } else if (mRenderer != NULL) {
774        uint32_t flags;
775        int64_t durationUs;
776        bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
777        if (getAudioDeepBufferSetting() // override regardless of source duration
778                || (mSource->getDuration(&durationUs) == OK
779                        && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
780            flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
781        } else {
782            flags = AUDIO_OUTPUT_FLAG_NONE;
783        }
784
785        sp<AMessage> reply = new AMessage(kWhatAudioOutputFormatChanged, this);
786        reply->setInt32("generation", mBufferGeneration);
787        mRenderer->changeAudioFormat(
788                format, false /* offloadOnly */, hasVideo,
789                flags, mSource->isStreaming(), reply);
790    }
791}
792
793void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
794    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
795        if (mMediaBuffers[i] != NULL) {
796            mMediaBuffers[i]->release();
797            mMediaBuffers.editItemAt(i) = NULL;
798        }
799    }
800    mMediaBuffers.resize(mInputBuffers.size());
801    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
802        mMediaBuffers.editItemAt(i) = NULL;
803    }
804    mInputBufferIsDequeued.clear();
805    mInputBufferIsDequeued.resize(mInputBuffers.size());
806    for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
807        mInputBufferIsDequeued.editItemAt(i) = false;
808    }
809
810    mPendingInputMessages.clear();
811    mDequeuedInputBuffers.clear();
812    mSkipRenderingUntilMediaTimeUs = -1;
813}
814
815void NuPlayer::Decoder::requestCodecNotification() {
816    if (mCodec != NULL) {
817        sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
818        reply->setInt32("generation", mBufferGeneration);
819        mCodec->requestActivityNotification(reply);
820    }
821}
822
823bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
824    int32_t generation;
825    CHECK(msg->findInt32("generation", &generation));
826    return generation != mBufferGeneration;
827}
828
829status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
830    sp<ABuffer> accessUnit;
831    bool dropAccessUnit = true;
832    do {
833        status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
834
835        if (err == -EWOULDBLOCK) {
836            return err;
837        } else if (err != OK) {
838            if (err == INFO_DISCONTINUITY) {
839                int32_t type;
840                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
841
842                bool formatChange =
843                    (mIsAudio &&
844                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
845                    || (!mIsAudio &&
846                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
847
848                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
849
850                ALOGI("%s discontinuity (format=%d, time=%d)",
851                        mIsAudio ? "audio" : "video", formatChange, timeChange);
852
853                bool seamlessFormatChange = false;
854                sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
855                if (formatChange) {
856                    seamlessFormatChange =
857                        supportsSeamlessFormatChange(newFormat);
858                    // treat seamless format change separately
859                    formatChange = !seamlessFormatChange;
860                }
861
862                // For format or time change, return EOS to queue EOS input,
863                // then wait for EOS on output.
864                if (formatChange /* not seamless */) {
865                    mFormatChangePending = true;
866                    err = ERROR_END_OF_STREAM;
867                } else if (timeChange) {
868                    rememberCodecSpecificData(newFormat);
869                    mTimeChangePending = true;
870                    err = ERROR_END_OF_STREAM;
871                } else if (seamlessFormatChange) {
872                    // reuse existing decoder and don't flush
873                    rememberCodecSpecificData(newFormat);
874                    continue;
875                } else {
876                    // This stream is unaffected by the discontinuity
877                    return -EWOULDBLOCK;
878                }
879            }
880
881            // reply should only be returned without a buffer set
882            // when there is an error (including EOS)
883            CHECK(err != OK);
884
885            reply->setInt32("err", err);
886            return ERROR_END_OF_STREAM;
887        }
888
889        dropAccessUnit = false;
890        if (!mIsAudio && !mIsEncrypted) {
891            // Extra safeguard if higher-level behavior changes. Otherwise, not required now.
892            // Preventing the buffer from being processed (and sent to codec) if this is a later
893            // round of playback but this time without prepareDrm. Or if there is a race between
894            // stop (which is not blocking) and releaseDrm allowing buffers being processed after
895            // Crypto has been released (GenericSource currently prevents this race though).
896            // Particularly doing this check before IsAVCReferenceFrame call to prevent parsing
897            // of encrypted data.
898            if (mIsEncryptedObservedEarlier) {
899                ALOGE("fetchInputData: mismatched mIsEncrypted/mIsEncryptedObservedEarlier (0/1)");
900
901                return INVALID_OPERATION;
902            }
903
904            int32_t layerId = 0;
905            bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId);
906            if (mRenderer->getVideoLateByUs() > 100000ll
907                    && mIsVideoAVC
908                    && !IsAVCReferenceFrame(accessUnit)) {
909                dropAccessUnit = true;
910            } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) {
911                // Add only one layer each time.
912                if (layerId > mCurrentMaxVideoTemporalLayerId + 1
913                        || layerId >= mNumVideoTemporalLayerAllowed) {
914                    dropAccessUnit = true;
915                    ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d",
916                            layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed,
917                            mCurrentMaxVideoTemporalLayerId);
918                } else if (layerId > mCurrentMaxVideoTemporalLayerId) {
919                    mCurrentMaxVideoTemporalLayerId = layerId;
920                } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) {
921                    mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1;
922                }
923            }
924            if (dropAccessUnit) {
925                if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) {
926                    mCurrentMaxVideoTemporalLayerId = layerId - 1;
927                }
928                ++mNumInputFramesDropped;
929            }
930        }
931    } while (dropAccessUnit);
932
933    // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
934#if 0
935    int64_t mediaTimeUs;
936    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
937    ALOGV("[%s] feeding input buffer at media time %.3f",
938         mIsAudio ? "audio" : "video",
939         mediaTimeUs / 1E6);
940#endif
941
942    if (mCCDecoder != NULL) {
943        mCCDecoder->decode(accessUnit);
944    }
945
946    reply->setBuffer("buffer", accessUnit);
947
948    return OK;
949}
950
951bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
952    size_t bufferIx;
953    CHECK(msg->findSize("buffer-ix", &bufferIx));
954    CHECK_LT(bufferIx, mInputBuffers.size());
955    sp<MediaCodecBuffer> codecBuffer = mInputBuffers[bufferIx];
956
957    sp<ABuffer> buffer;
958    bool hasBuffer = msg->findBuffer("buffer", &buffer);
959    bool needsCopy = true;
960
961    if (buffer == NULL /* includes !hasBuffer */) {
962        int32_t streamErr = ERROR_END_OF_STREAM;
963        CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
964
965        CHECK(streamErr != OK);
966
967        // attempt to queue EOS
968        status_t err = mCodec->queueInputBuffer(
969                bufferIx,
970                0,
971                0,
972                0,
973                MediaCodec::BUFFER_FLAG_EOS);
974        if (err == OK) {
975            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
976        } else if (streamErr == ERROR_END_OF_STREAM) {
977            streamErr = err;
978            // err will not be ERROR_END_OF_STREAM
979        }
980
981        if (streamErr != ERROR_END_OF_STREAM) {
982            ALOGE("Stream error for %s (err=%d), EOS %s queued",
983                    mComponentName.c_str(),
984                    streamErr,
985                    err == OK ? "successfully" : "unsuccessfully");
986            handleError(streamErr);
987        }
988    } else {
989        sp<AMessage> extra;
990        if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
991            int64_t resumeAtMediaTimeUs;
992            if (extra->findInt64(
993                        "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
994                ALOGI("[%s] suppressing rendering until %lld us",
995                        mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
996                mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
997            }
998        }
999
1000        int64_t timeUs = 0;
1001        uint32_t flags = 0;
1002        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1003
1004        int32_t eos, csd;
1005        // we do not expect SYNCFRAME for decoder
1006        if (buffer->meta()->findInt32("eos", &eos) && eos) {
1007            flags |= MediaCodec::BUFFER_FLAG_EOS;
1008        } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
1009            flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
1010        }
1011
1012        // Modular DRM
1013        MediaBuffer *mediaBuf = NULL;
1014        NuPlayerDrm::CryptoInfo *cryptInfo = NULL;
1015
1016        // copy into codec buffer
1017        if (needsCopy) {
1018            if (buffer->size() > codecBuffer->capacity()) {
1019                handleError(ERROR_BUFFER_TOO_SMALL);
1020                mDequeuedInputBuffers.push_back(bufferIx);
1021                return false;
1022            }
1023
1024            if (buffer->data() != NULL) {
1025                codecBuffer->setRange(0, buffer->size());
1026                memcpy(codecBuffer->data(), buffer->data(), buffer->size());
1027            } else { // No buffer->data()
1028                //Modular DRM
1029                mediaBuf = (MediaBuffer*)buffer->getMediaBufferBase();
1030                if (mediaBuf != NULL) {
1031                    codecBuffer->setRange(0, mediaBuf->size());
1032                    memcpy(codecBuffer->data(), mediaBuf->data(), mediaBuf->size());
1033
1034                    sp<MetaData> meta_data = mediaBuf->meta_data();
1035                    cryptInfo = NuPlayerDrm::getSampleCryptoInfo(meta_data);
1036
1037                    // since getMediaBuffer() has incremented the refCount
1038                    mediaBuf->release();
1039                } else { // No mediaBuf
1040                    ALOGE("onInputBufferFetched: buffer->data()/mediaBuf are NULL for %p",
1041                            buffer.get());
1042                    handleError(UNKNOWN_ERROR);
1043                    return false;
1044                }
1045            } // buffer->data()
1046        } // needsCopy
1047
1048        status_t err;
1049        AString errorDetailMsg;
1050        if (cryptInfo != NULL) {
1051            err = mCodec->queueSecureInputBuffer(
1052                    bufferIx,
1053                    codecBuffer->offset(),
1054                    cryptInfo->subSamples,
1055                    cryptInfo->numSubSamples,
1056                    cryptInfo->key,
1057                    cryptInfo->iv,
1058                    cryptInfo->mode,
1059                    cryptInfo->pattern,
1060                    timeUs,
1061                    flags,
1062                    &errorDetailMsg);
1063            // synchronous call so done with cryptInfo here
1064            free(cryptInfo);
1065        } else {
1066            err = mCodec->queueInputBuffer(
1067                    bufferIx,
1068                    codecBuffer->offset(),
1069                    codecBuffer->size(),
1070                    timeUs,
1071                    flags,
1072                    &errorDetailMsg);
1073        } // no cryptInfo
1074
1075        if (err != OK) {
1076            ALOGE("onInputBufferFetched: queue%sInputBuffer failed for %s (err=%d, %s)",
1077                    (cryptInfo != NULL ? "Secure" : ""),
1078                    mComponentName.c_str(), err, errorDetailMsg.c_str());
1079            handleError(err);
1080        } else {
1081            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
1082        }
1083
1084    }   // buffer != NULL
1085    return true;
1086}
1087
1088void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
1089    status_t err;
1090    int32_t render;
1091    size_t bufferIx;
1092    int32_t eos;
1093    CHECK(msg->findSize("buffer-ix", &bufferIx));
1094
1095    if (!mIsAudio) {
1096        int64_t timeUs;
1097        sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx];
1098        buffer->meta()->findInt64("timeUs", &timeUs);
1099
1100        if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
1101            mCCDecoder->display(timeUs);
1102        }
1103    }
1104
1105    if (msg->findInt32("render", &render) && render) {
1106        int64_t timestampNs;
1107        CHECK(msg->findInt64("timestampNs", &timestampNs));
1108        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
1109    } else {
1110        mNumOutputFramesDropped += !mIsAudio;
1111        err = mCodec->releaseOutputBuffer(bufferIx);
1112    }
1113    if (err != OK) {
1114        ALOGE("failed to release output buffer for %s (err=%d)",
1115                mComponentName.c_str(), err);
1116        handleError(err);
1117    }
1118    if (msg->findInt32("eos", &eos) && eos
1119            && isDiscontinuityPending()) {
1120        finishHandleDiscontinuity(true /* flushOnTimeChange */);
1121    }
1122}
1123
1124bool NuPlayer::Decoder::isDiscontinuityPending() const {
1125    return mFormatChangePending || mTimeChangePending;
1126}
1127
1128void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
1129    ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
1130            mFormatChangePending, mTimeChangePending, flushOnTimeChange);
1131
1132    // If we have format change, pause and wait to be killed;
1133    // If we have time change only, flush and restart fetching.
1134
1135    if (mFormatChangePending) {
1136        mPaused = true;
1137    } else if (mTimeChangePending) {
1138        if (flushOnTimeChange) {
1139            doFlush(false /* notifyComplete */);
1140            signalResume(false /* notifyComplete */);
1141        }
1142    }
1143
1144    // Notify NuPlayer to either shutdown decoder, or rescan sources
1145    sp<AMessage> msg = mNotify->dup();
1146    msg->setInt32("what", kWhatInputDiscontinuity);
1147    msg->setInt32("formatChange", mFormatChangePending);
1148    msg->post();
1149
1150    mFormatChangePending = false;
1151    mTimeChangePending = false;
1152}
1153
1154bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
1155        const sp<AMessage> &targetFormat) const {
1156    if (targetFormat == NULL) {
1157        return true;
1158    }
1159
1160    AString mime;
1161    if (!targetFormat->findString("mime", &mime)) {
1162        return false;
1163    }
1164
1165    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
1166        // field-by-field comparison
1167        const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
1168        for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
1169            int32_t oldVal, newVal;
1170            if (!mInputFormat->findInt32(keys[i], &oldVal) ||
1171                    !targetFormat->findInt32(keys[i], &newVal) ||
1172                    oldVal != newVal) {
1173                return false;
1174            }
1175        }
1176
1177        sp<ABuffer> oldBuf, newBuf;
1178        if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
1179                targetFormat->findBuffer("csd-0", &newBuf)) {
1180            if (oldBuf->size() != newBuf->size()) {
1181                return false;
1182            }
1183            return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
1184        }
1185    }
1186    return false;
1187}
1188
1189bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
1190    if (mInputFormat == NULL) {
1191        return false;
1192    }
1193
1194    if (targetFormat == NULL) {
1195        return true;
1196    }
1197
1198    AString oldMime, newMime;
1199    if (!mInputFormat->findString("mime", &oldMime)
1200            || !targetFormat->findString("mime", &newMime)
1201            || !(oldMime == newMime)) {
1202        return false;
1203    }
1204
1205    bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
1206    bool seamless;
1207    if (audio) {
1208        seamless = supportsSeamlessAudioFormatChange(targetFormat);
1209    } else {
1210        int32_t isAdaptive;
1211        seamless = (mCodec != NULL &&
1212                mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
1213                isAdaptive);
1214    }
1215
1216    ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
1217    return seamless;
1218}
1219
1220void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
1221    if (format == NULL) {
1222        return;
1223    }
1224    mCSDsForCurrentFormat.clear();
1225    for (int32_t i = 0; ; ++i) {
1226        AString tag = "csd-";
1227        tag.append(i);
1228        sp<ABuffer> buffer;
1229        if (!format->findBuffer(tag.c_str(), &buffer)) {
1230            break;
1231        }
1232        mCSDsForCurrentFormat.push(buffer);
1233    }
1234}
1235
1236void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
1237    if (mResumePending) {
1238        mResumePending = false;
1239
1240        sp<AMessage> notify = mNotify->dup();
1241        notify->setInt32("what", kWhatResumeCompleted);
1242        notify->post();
1243    }
1244}
1245
1246}  // namespace android
1247
1248