NuPlayerDecoder.cpp revision c387f2b719a1a26c8306f77d79cc9a6f26b36813 
1/*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerDecoder"
19#include <utils/Log.h>
20#include <inttypes.h>
21
22#include <algorithm>
23
24#include "NuPlayerCCDecoder.h"
25#include "NuPlayerDecoder.h"
26#include "NuPlayerDrm.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29
30#include <cutils/properties.h>
31#include <media/ICrypto.h>
32#include <media/MediaCodecBuffer.h>
33#include <media/stagefright/foundation/ABuffer.h>
34#include <media/stagefright/foundation/ADebug.h>
35#include <media/stagefright/foundation/AMessage.h>
36#include <media/stagefright/MediaBuffer.h>
37#include <media/stagefright/MediaCodec.h>
38#include <media/stagefright/MediaDefs.h>
39#include <media/stagefright/MediaErrors.h>
40#include <media/stagefright/SurfaceUtils.h>
41#include <gui/Surface.h>
42
43#include "avc_utils.h"
44#include "ATSParser.h"
45
46namespace android {
47
48static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display
49
50// The default total video frame rate of a stream when that info is not available from
51// the source.
52static float kDefaultVideoFrameRateTotal = 30.f;
53
54static inline bool getAudioDeepBufferSetting() {
55    return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
56}
57
58NuPlayer::Decoder::Decoder(
59        const sp<AMessage> &notify,
60        const sp<Source> &source,
61        pid_t pid,
62        uid_t uid,
63        const sp<Renderer> &renderer,
64        const sp<Surface> &surface,
65        const sp<CCDecoder> &ccDecoder)
66    : DecoderBase(notify),
67      mSurface(surface),
68      mSource(source),
69      mRenderer(renderer),
70      mCCDecoder(ccDecoder),
71      mPid(pid),
72      mUid(uid),
73      mSkipRenderingUntilMediaTimeUs(-1ll),
74      mNumFramesTotal(0ll),
75      mNumInputFramesDropped(0ll),
76      mNumOutputFramesDropped(0ll),
77      mVideoWidth(0),
78      mVideoHeight(0),
79      mIsAudio(true),
80      mIsVideoAVC(false),
81      mIsSecure(false),
82      mFormatChangePending(false),
83      mTimeChangePending(false),
84      mFrameRateTotal(kDefaultVideoFrameRateTotal),
85      mPlaybackSpeed(1.0f),
86      mNumVideoTemporalLayerTotal(1), // decode all layers
87      mNumVideoTemporalLayerAllowed(1),
88      mCurrentMaxVideoTemporalLayerId(0),
89      mResumePending(false),
90      mComponentName("decoder") {
91    mCodecLooper = new ALooper;
92    mCodecLooper->setName("NPDecoder-CL");
93    mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
94    mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;
95}
96
97NuPlayer::Decoder::~Decoder() {
98    mCodec->release();
99    releaseAndResetMediaBuffers();
100}
101
102sp<AMessage> NuPlayer::Decoder::getStats() const {
103    mStats->setInt64("frames-total", mNumFramesTotal);
104    mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
105    mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
106    return mStats;
107}
108
109status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
110    if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
111        return BAD_VALUE;
112    }
113
114    sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
115
116    msg->setObject("surface", surface);
117    sp<AMessage> response;
118    status_t err = msg->postAndAwaitResponse(&response);
119    if (err == OK && response != NULL) {
120        CHECK(response->findInt32("err", &err));
121    }
122    return err;
123}
124
125void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
126    ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
127
128    switch (msg->what()) {
129        case kWhatCodecNotify:
130        {
131            int32_t cbID;
132            CHECK(msg->findInt32("callbackID", &cbID));
133
134            ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
135                    mIsAudio ? "audio" : "video", cbID, mPaused);
136
137            if (mPaused) {
138                break;
139            }
140
141            switch (cbID) {
142                case MediaCodec::CB_INPUT_AVAILABLE:
143                {
144                    int32_t index;
145                    CHECK(msg->findInt32("index", &index));
146
147                    handleAnInputBuffer(index);
148                    break;
149                }
150
151                case MediaCodec::CB_OUTPUT_AVAILABLE:
152                {
153                    int32_t index;
154                    size_t offset;
155                    size_t size;
156                    int64_t timeUs;
157                    int32_t flags;
158
159                    CHECK(msg->findInt32("index", &index));
160                    CHECK(msg->findSize("offset", &offset));
161                    CHECK(msg->findSize("size", &size));
162                    CHECK(msg->findInt64("timeUs", &timeUs));
163                    CHECK(msg->findInt32("flags", &flags));
164
165                    handleAnOutputBuffer(index, offset, size, timeUs, flags);
166                    break;
167                }
168
169                case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
170                {
171                    sp<AMessage> format;
172                    CHECK(msg->findMessage("format", &format));
173
174                    handleOutputFormatChange(format);
175                    break;
176                }
177
178                case MediaCodec::CB_ERROR:
179                {
180                    status_t err;
181                    CHECK(msg->findInt32("err", &err));
182                    ALOGE("Decoder (%s) reported error : 0x%x",
183                            mIsAudio ? "audio" : "video", err);
184
185                    handleError(err);
186                    break;
187                }
188
189                default:
190                {
191                    TRESPASS();
192                    break;
193                }
194            }
195
196            break;
197        }
198
199        case kWhatRenderBuffer:
200        {
201            if (!isStaleReply(msg)) {
202                onRenderBuffer(msg);
203            }
204            break;
205        }
206
207        case kWhatAudioOutputFormatChanged:
208        {
209            if (!isStaleReply(msg)) {
210                status_t err;
211                if (msg->findInt32("err", &err) && err != OK) {
212                    ALOGE("Renderer reported 0x%x when changing audio output format", err);
213                    handleError(err);
214                }
215            }
216            break;
217        }
218
219        case kWhatSetVideoSurface:
220        {
221            sp<AReplyToken> replyID;
222            CHECK(msg->senderAwaitsResponse(&replyID));
223
224            sp<RefBase> obj;
225            CHECK(msg->findObject("surface", &obj));
226            sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
227            int32_t err = INVALID_OPERATION;
228            // NOTE: in practice mSurface is always non-null, but checking here for completeness
229            if (mCodec != NULL && mSurface != NULL) {
230                // TODO: once AwesomePlayer is removed, remove this automatic connecting
231                // to the surface by MediaPlayerService.
232                //
233                // at this point MediaPlayerService::client has already connected to the
234                // surface, which MediaCodec does not expect
235                err = nativeWindowDisconnect(surface.get(), "kWhatSetVideoSurface(surface)");
236                if (err == OK) {
237                    err = mCodec->setSurface(surface);
238                    ALOGI_IF(err, "codec setSurface returned: %d", err);
239                    if (err == OK) {
240                        // reconnect to the old surface as MPS::Client will expect to
241                        // be able to disconnect from it.
242                        (void)nativeWindowConnect(mSurface.get(), "kWhatSetVideoSurface(mSurface)");
243                        mSurface = surface;
244                    }
245                }
246                if (err != OK) {
247                    // reconnect to the new surface on error as MPS::Client will expect to
248                    // be able to disconnect from it.
249                    (void)nativeWindowConnect(surface.get(), "kWhatSetVideoSurface(err)");
250                }
251            }
252
253            sp<AMessage> response = new AMessage;
254            response->setInt32("err", err);
255            response->postReply(replyID);
256            break;
257        }
258
259        case kWhatDrmReleaseCrypto:
260        {
261            ALOGV("kWhatDrmReleaseCrypto");
262            onReleaseCrypto(msg);
263            break;
264        }
265
266        default:
267            DecoderBase::onMessageReceived(msg);
268            break;
269    }
270}
271
272void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
273    CHECK(mCodec == NULL);
274
275    mFormatChangePending = false;
276    mTimeChangePending = false;
277
278    ++mBufferGeneration;
279
280    AString mime;
281    CHECK(format->findString("mime", &mime));
282
283    mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
284    mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
285
286    mComponentName = mime;
287    mComponentName.append(" decoder");
288    ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
289
290    mCodec = MediaCodec::CreateByType(
291            mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid);
292    int32_t secure = 0;
293    if (format->findInt32("secure", &secure) && secure != 0) {
294        if (mCodec != NULL) {
295            mCodec->getName(&mComponentName);
296            mComponentName.append(".secure");
297            mCodec->release();
298            ALOGI("[%s] creating", mComponentName.c_str());
299            mCodec = MediaCodec::CreateByComponentName(
300                    mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid);
301        }
302    }
303    if (mCodec == NULL) {
304        ALOGE("Failed to create %s%s decoder",
305                (secure ? "secure " : ""), mime.c_str());
306        handleError(UNKNOWN_ERROR);
307        return;
308    }
309    mIsSecure = secure;
310
311    mCodec->getName(&mComponentName);
312
313    status_t err;
314    if (mSurface != NULL) {
315        // disconnect from surface as MediaCodec will reconnect
316        err = nativeWindowDisconnect(mSurface.get(), "onConfigure");
317        // We treat this as a warning, as this is a preparatory step.
318        // Codec will try to connect to the surface, which is where
319        // any error signaling will occur.
320        ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
321    }
322
323    // Modular DRM
324    void *pCrypto;
325    if (!format->findPointer("crypto", &pCrypto)) {
326        pCrypto = NULL;
327    }
328    sp<ICrypto> crypto = (ICrypto*)pCrypto;
329    ALOGV("onConfigure mCrypto: %p (%d)  mIsSecure: %d",
330            crypto.get(), (crypto != NULL ? crypto->getStrongCount() : 0), mIsSecure);
331
332    err = mCodec->configure(
333            format, mSurface, crypto, 0 /* flags */);
334
335    if (err != OK) {
336        ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
337        mCodec->release();
338        mCodec.clear();
339        handleError(err);
340        return;
341    }
342    rememberCodecSpecificData(format);
343
344    // the following should work in configured state
345    CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
346    CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
347
348    mStats->setString("mime", mime.c_str());
349    mStats->setString("component-name", mComponentName.c_str());
350
351    if (!mIsAudio) {
352        int32_t width, height;
353        if (mOutputFormat->findInt32("width", &width)
354                && mOutputFormat->findInt32("height", &height)) {
355            mStats->setInt32("width", width);
356            mStats->setInt32("height", height);
357        }
358    }
359
360    sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
361    mCodec->setCallback(reply);
362
363    err = mCodec->start();
364    if (err != OK) {
365        ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
366        mCodec->release();
367        mCodec.clear();
368        handleError(err);
369        return;
370    }
371
372    releaseAndResetMediaBuffers();
373
374    mPaused = false;
375    mResumePending = false;
376}
377
378void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) {
379    bool needAdjustLayers = false;
380    float frameRateTotal;
381    if (params->findFloat("frame-rate-total", &frameRateTotal)
382            && mFrameRateTotal != frameRateTotal) {
383        needAdjustLayers = true;
384        mFrameRateTotal = frameRateTotal;
385    }
386
387    int32_t numVideoTemporalLayerTotal;
388    if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal)
389            && numVideoTemporalLayerTotal >= 0
390            && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers
391            && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) {
392        needAdjustLayers = true;
393        mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1);
394    }
395
396    if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) {
397        // TODO: For now, layer fps is calculated for some specific architectures.
398        // But it really should be extracted from the stream.
399        mVideoTemporalLayerAggregateFps[0] =
400            mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1));
401        for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) {
402            mVideoTemporalLayerAggregateFps[i] =
403                mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i))
404                + mVideoTemporalLayerAggregateFps[i - 1];
405        }
406    }
407
408    float playbackSpeed;
409    if (params->findFloat("playback-speed", &playbackSpeed)
410            && mPlaybackSpeed != playbackSpeed) {
411        needAdjustLayers = true;
412        mPlaybackSpeed = playbackSpeed;
413    }
414
415    if (needAdjustLayers) {
416        float decodeFrameRate = mFrameRateTotal;
417        // enable temporal layering optimization only if we know the layering depth
418        if (mNumVideoTemporalLayerTotal > 1) {
419            int32_t layerId;
420            for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) {
421                if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed
422                        >= kDisplayRefreshingRate * 0.9) {
423                    break;
424                }
425            }
426            mNumVideoTemporalLayerAllowed = layerId + 1;
427            decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId];
428        }
429        ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g",
430                mNumVideoTemporalLayerAllowed, decodeFrameRate);
431
432        if (mCodec == NULL) {
433            ALOGW("onSetParameters called before codec is created.");
434            return;
435        }
436
437        sp<AMessage> codecParams = new AMessage();
438        codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed);
439        mCodec->setParameters(codecParams);
440    }
441}
442
443void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
444    mRenderer = renderer;
445}
446
447void NuPlayer::Decoder::onResume(bool notifyComplete) {
448    mPaused = false;
449
450    if (notifyComplete) {
451        mResumePending = true;
452    }
453    mCodec->start();
454}
455
456void NuPlayer::Decoder::doFlush(bool notifyComplete) {
457    if (mCCDecoder != NULL) {
458        mCCDecoder->flush();
459    }
460
461    if (mRenderer != NULL) {
462        mRenderer->flush(mIsAudio, notifyComplete);
463        mRenderer->signalTimeDiscontinuity();
464    }
465
466    status_t err = OK;
467    if (mCodec != NULL) {
468        err = mCodec->flush();
469        mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
470        ++mBufferGeneration;
471    }
472
473    if (err != OK) {
474        ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
475        handleError(err);
476        // finish with posting kWhatFlushCompleted.
477        // we attempt to release the buffers even if flush fails.
478    }
479    releaseAndResetMediaBuffers();
480    mPaused = true;
481}
482
483
484void NuPlayer::Decoder::onFlush() {
485    doFlush(true);
486
487    if (isDiscontinuityPending()) {
488        // This could happen if the client starts seeking/shutdown
489        // after we queued an EOS for discontinuities.
490        // We can consider discontinuity handled.
491        finishHandleDiscontinuity(false /* flushOnTimeChange */);
492    }
493
494    sp<AMessage> notify = mNotify->dup();
495    notify->setInt32("what", kWhatFlushCompleted);
496    notify->post();
497}
498
499void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
500    status_t err = OK;
501
502    // if there is a pending resume request, notify complete now
503    notifyResumeCompleteIfNecessary();
504
505    if (mCodec != NULL) {
506        err = mCodec->release();
507        mCodec = NULL;
508        ++mBufferGeneration;
509
510        if (mSurface != NULL) {
511            // reconnect to surface as MediaCodec disconnected from it
512            status_t error = nativeWindowConnect(mSurface.get(), "onShutdown");
513            ALOGW_IF(error != NO_ERROR,
514                    "[%s] failed to connect to native window, error=%d",
515                    mComponentName.c_str(), error);
516        }
517        mComponentName = "decoder";
518    }
519
520    releaseAndResetMediaBuffers();
521
522    if (err != OK) {
523        ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
524        handleError(err);
525        // finish with posting kWhatShutdownCompleted.
526    }
527
528    if (notifyComplete) {
529        sp<AMessage> notify = mNotify->dup();
530        notify->setInt32("what", kWhatShutdownCompleted);
531        notify->post();
532        mPaused = true;
533    }
534}
535
536/*
537 * returns true if we should request more data
538 */
539bool NuPlayer::Decoder::doRequestBuffers() {
540    if (isDiscontinuityPending()) {
541        return false;
542    }
543    status_t err = OK;
544    while (err == OK && !mDequeuedInputBuffers.empty()) {
545        size_t bufferIx = *mDequeuedInputBuffers.begin();
546        sp<AMessage> msg = new AMessage();
547        msg->setSize("buffer-ix", bufferIx);
548        err = fetchInputData(msg);
549        if (err != OK && err != ERROR_END_OF_STREAM) {
550            // if EOS, need to queue EOS buffer
551            break;
552        }
553        mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
554
555        if (!mPendingInputMessages.empty()
556                || !onInputBufferFetched(msg)) {
557            mPendingInputMessages.push_back(msg);
558        }
559    }
560
561    return err == -EWOULDBLOCK
562            && mSource->feedMoreTSData() == OK;
563}
564
565void NuPlayer::Decoder::handleError(int32_t err)
566{
567    // We cannot immediately release the codec due to buffers still outstanding
568    // in the renderer.  We signal to the player the error so it can shutdown/release the
569    // decoder after flushing and increment the generation to discard unnecessary messages.
570
571    ++mBufferGeneration;
572
573    sp<AMessage> notify = mNotify->dup();
574    notify->setInt32("what", kWhatError);
575    notify->setInt32("err", err);
576    notify->post();
577}
578
579status_t NuPlayer::Decoder::releaseCrypto()
580{
581    ALOGV("releaseCrypto");
582
583    sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this);
584
585    sp<AMessage> response;
586    status_t status = msg->postAndAwaitResponse(&response);
587    if (status == OK && response != NULL) {
588        CHECK(response->findInt32("status", &status));
589        ALOGV("releaseCrypto ret: %d ", status);
590    } else {
591        ALOGE("releaseCrypto err: %d", status);
592    }
593
594    return status;
595}
596
597void NuPlayer::Decoder::onReleaseCrypto(const sp<AMessage>& msg)
598{
599    status_t status = INVALID_OPERATION;
600    if (mCodec != NULL) {
601        status = mCodec->releaseCrypto();
602    } else {
603        // returning OK if the codec has been already released
604        status = OK;
605        ALOGE("onReleaseCrypto No mCodec. err: %d", status);
606    }
607
608    sp<AMessage> response = new AMessage;
609    response->setInt32("status", status);
610
611    sp<AReplyToken> replyID;
612    CHECK(msg->senderAwaitsResponse(&replyID));
613    response->postReply(replyID);
614}
615
616bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
617    if (isDiscontinuityPending()) {
618        return false;
619    }
620
621    sp<MediaCodecBuffer> buffer;
622    mCodec->getInputBuffer(index, &buffer);
623
624    if (buffer == NULL) {
625        handleError(UNKNOWN_ERROR);
626        return false;
627    }
628
629    if (index >= mInputBuffers.size()) {
630        for (size_t i = mInputBuffers.size(); i <= index; ++i) {
631            mInputBuffers.add();
632            mMediaBuffers.add();
633            mInputBufferIsDequeued.add();
634            mMediaBuffers.editItemAt(i) = NULL;
635            mInputBufferIsDequeued.editItemAt(i) = false;
636        }
637    }
638    mInputBuffers.editItemAt(index) = buffer;
639
640    //CHECK_LT(bufferIx, mInputBuffers.size());
641
642    if (mMediaBuffers[index] != NULL) {
643        mMediaBuffers[index]->release();
644        mMediaBuffers.editItemAt(index) = NULL;
645    }
646    mInputBufferIsDequeued.editItemAt(index) = true;
647
648    if (!mCSDsToSubmit.isEmpty()) {
649        sp<AMessage> msg = new AMessage();
650        msg->setSize("buffer-ix", index);
651
652        sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
653        ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
654        msg->setBuffer("buffer", buffer);
655        mCSDsToSubmit.removeAt(0);
656        if (!onInputBufferFetched(msg)) {
657            handleError(UNKNOWN_ERROR);
658            return false;
659        }
660        return true;
661    }
662
663    while (!mPendingInputMessages.empty()) {
664        sp<AMessage> msg = *mPendingInputMessages.begin();
665        if (!onInputBufferFetched(msg)) {
666            break;
667        }
668        mPendingInputMessages.erase(mPendingInputMessages.begin());
669    }
670
671    if (!mInputBufferIsDequeued.editItemAt(index)) {
672        return true;
673    }
674
675    mDequeuedInputBuffers.push_back(index);
676
677    onRequestInputBuffers();
678    return true;
679}
680
681bool NuPlayer::Decoder::handleAnOutputBuffer(
682        size_t index,
683        size_t offset,
684        size_t size,
685        int64_t timeUs,
686        int32_t flags) {
687//    CHECK_LT(bufferIx, mOutputBuffers.size());
688    sp<MediaCodecBuffer> buffer;
689    mCodec->getOutputBuffer(index, &buffer);
690
691    if (buffer == NULL) {
692        handleError(UNKNOWN_ERROR);
693        return false;
694    }
695
696    if (index >= mOutputBuffers.size()) {
697        for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
698            mOutputBuffers.add();
699        }
700    }
701
702    mOutputBuffers.editItemAt(index) = buffer;
703
704    buffer->setRange(offset, size);
705    buffer->meta()->clear();
706    buffer->meta()->setInt64("timeUs", timeUs);
707
708    bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
709    // we do not expect CODECCONFIG or SYNCFRAME for decoder
710
711    sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
712    reply->setSize("buffer-ix", index);
713    reply->setInt32("generation", mBufferGeneration);
714
715    if (eos) {
716        ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
717
718        buffer->meta()->setInt32("eos", true);
719        reply->setInt32("eos", true);
720    } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
721        if (timeUs < mSkipRenderingUntilMediaTimeUs) {
722            ALOGV("[%s] dropping buffer at time %lld as requested.",
723                     mComponentName.c_str(), (long long)timeUs);
724
725            reply->post();
726            return true;
727        }
728
729        mSkipRenderingUntilMediaTimeUs = -1;
730    }
731
732    mNumFramesTotal += !mIsAudio;
733
734    // wait until 1st frame comes out to signal resume complete
735    notifyResumeCompleteIfNecessary();
736
737    if (mRenderer != NULL) {
738        // send the buffer to renderer.
739        mRenderer->queueBuffer(mIsAudio, buffer, reply);
740        if (eos && !isDiscontinuityPending()) {
741            mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
742        }
743    }
744
745    return true;
746}
747
748void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
749    if (!mIsAudio) {
750        int32_t width, height;
751        if (format->findInt32("width", &width)
752                && format->findInt32("height", &height)) {
753            mStats->setInt32("width", width);
754            mStats->setInt32("height", height);
755        }
756        sp<AMessage> notify = mNotify->dup();
757        notify->setInt32("what", kWhatVideoSizeChanged);
758        notify->setMessage("format", format);
759        notify->post();
760    } else if (mRenderer != NULL) {
761        uint32_t flags;
762        int64_t durationUs;
763        bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
764        if (getAudioDeepBufferSetting() // override regardless of source duration
765                || (!hasVideo
766                        && mSource->getDuration(&durationUs) == OK
767                        && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
768            flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
769        } else {
770            flags = AUDIO_OUTPUT_FLAG_NONE;
771        }
772
773        sp<AMessage> reply = new AMessage(kWhatAudioOutputFormatChanged, this);
774        reply->setInt32("generation", mBufferGeneration);
775        mRenderer->changeAudioFormat(
776                format, false /* offloadOnly */, hasVideo,
777                flags, mSource->isStreaming(), reply);
778    }
779}
780
781void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
782    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
783        if (mMediaBuffers[i] != NULL) {
784            mMediaBuffers[i]->release();
785            mMediaBuffers.editItemAt(i) = NULL;
786        }
787    }
788    mMediaBuffers.resize(mInputBuffers.size());
789    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
790        mMediaBuffers.editItemAt(i) = NULL;
791    }
792    mInputBufferIsDequeued.clear();
793    mInputBufferIsDequeued.resize(mInputBuffers.size());
794    for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
795        mInputBufferIsDequeued.editItemAt(i) = false;
796    }
797
798    mPendingInputMessages.clear();
799    mDequeuedInputBuffers.clear();
800    mSkipRenderingUntilMediaTimeUs = -1;
801}
802
803void NuPlayer::Decoder::requestCodecNotification() {
804    if (mCodec != NULL) {
805        sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
806        reply->setInt32("generation", mBufferGeneration);
807        mCodec->requestActivityNotification(reply);
808    }
809}
810
811bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
812    int32_t generation;
813    CHECK(msg->findInt32("generation", &generation));
814    return generation != mBufferGeneration;
815}
816
817status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
818    sp<ABuffer> accessUnit;
819    bool dropAccessUnit = true;
820    do {
821        status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
822
823        if (err == -EWOULDBLOCK) {
824            return err;
825        } else if (err != OK) {
826            if (err == INFO_DISCONTINUITY) {
827                int32_t type;
828                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
829
830                bool formatChange =
831                    (mIsAudio &&
832                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
833                    || (!mIsAudio &&
834                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
835
836                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
837
838                ALOGI("%s discontinuity (format=%d, time=%d)",
839                        mIsAudio ? "audio" : "video", formatChange, timeChange);
840
841                bool seamlessFormatChange = false;
842                sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
843                if (formatChange) {
844                    seamlessFormatChange =
845                        supportsSeamlessFormatChange(newFormat);
846                    // treat seamless format change separately
847                    formatChange = !seamlessFormatChange;
848                }
849
850                // For format or time change, return EOS to queue EOS input,
851                // then wait for EOS on output.
852                if (formatChange /* not seamless */) {
853                    mFormatChangePending = true;
854                    err = ERROR_END_OF_STREAM;
855                } else if (timeChange) {
856                    rememberCodecSpecificData(newFormat);
857                    mTimeChangePending = true;
858                    err = ERROR_END_OF_STREAM;
859                } else if (seamlessFormatChange) {
860                    // reuse existing decoder and don't flush
861                    rememberCodecSpecificData(newFormat);
862                    continue;
863                } else {
864                    // This stream is unaffected by the discontinuity
865                    return -EWOULDBLOCK;
866                }
867            }
868
869            // reply should only be returned without a buffer set
870            // when there is an error (including EOS)
871            CHECK(err != OK);
872
873            reply->setInt32("err", err);
874            return ERROR_END_OF_STREAM;
875        }
876
877        dropAccessUnit = false;
878        if (!mIsAudio && !mIsSecure) {
879            int32_t layerId = 0;
880            bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId);
881            if (mRenderer->getVideoLateByUs() > 100000ll
882                    && mIsVideoAVC
883                    && !IsAVCReferenceFrame(accessUnit)) {
884                dropAccessUnit = true;
885            } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) {
886                // Add only one layer each time.
887                if (layerId > mCurrentMaxVideoTemporalLayerId + 1
888                        || layerId >= mNumVideoTemporalLayerAllowed) {
889                    dropAccessUnit = true;
890                    ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d",
891                            layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed,
892                            mCurrentMaxVideoTemporalLayerId);
893                } else if (layerId > mCurrentMaxVideoTemporalLayerId) {
894                    mCurrentMaxVideoTemporalLayerId = layerId;
895                } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) {
896                    mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1;
897                }
898            }
899            if (dropAccessUnit) {
900                if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) {
901                    mCurrentMaxVideoTemporalLayerId = layerId - 1;
902                }
903                ++mNumInputFramesDropped;
904            }
905        }
906    } while (dropAccessUnit);
907
908    // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
909#if 0
910    int64_t mediaTimeUs;
911    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
912    ALOGV("[%s] feeding input buffer at media time %.3f",
913         mIsAudio ? "audio" : "video",
914         mediaTimeUs / 1E6);
915#endif
916
917    if (mCCDecoder != NULL) {
918        mCCDecoder->decode(accessUnit);
919    }
920
921    reply->setBuffer("buffer", accessUnit);
922
923    return OK;
924}
925
926bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
927    size_t bufferIx;
928    CHECK(msg->findSize("buffer-ix", &bufferIx));
929    CHECK_LT(bufferIx, mInputBuffers.size());
930    sp<MediaCodecBuffer> codecBuffer = mInputBuffers[bufferIx];
931
932    sp<ABuffer> buffer;
933    bool hasBuffer = msg->findBuffer("buffer", &buffer);
934    bool needsCopy = true;
935
936    if (buffer == NULL /* includes !hasBuffer */) {
937        int32_t streamErr = ERROR_END_OF_STREAM;
938        CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
939
940        CHECK(streamErr != OK);
941
942        // attempt to queue EOS
943        status_t err = mCodec->queueInputBuffer(
944                bufferIx,
945                0,
946                0,
947                0,
948                MediaCodec::BUFFER_FLAG_EOS);
949        if (err == OK) {
950            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
951        } else if (streamErr == ERROR_END_OF_STREAM) {
952            streamErr = err;
953            // err will not be ERROR_END_OF_STREAM
954        }
955
956        if (streamErr != ERROR_END_OF_STREAM) {
957            ALOGE("Stream error for %s (err=%d), EOS %s queued",
958                    mComponentName.c_str(),
959                    streamErr,
960                    err == OK ? "successfully" : "unsuccessfully");
961            handleError(streamErr);
962        }
963    } else {
964        sp<AMessage> extra;
965        if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
966            int64_t resumeAtMediaTimeUs;
967            if (extra->findInt64(
968                        "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
969                ALOGI("[%s] suppressing rendering until %lld us",
970                        mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
971                mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
972            }
973        }
974
975        int64_t timeUs = 0;
976        uint32_t flags = 0;
977        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
978
979        int32_t eos, csd;
980        // we do not expect SYNCFRAME for decoder
981        if (buffer->meta()->findInt32("eos", &eos) && eos) {
982            flags |= MediaCodec::BUFFER_FLAG_EOS;
983        } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
984            flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
985        }
986
987        // Modular DRM
988        MediaBuffer *mediaBuf = NULL;
989        NuPlayerDrm::CryptoInfo *cryptInfo = NULL;
990
991        // copy into codec buffer
992        if (needsCopy) {
993            if (buffer->size() > codecBuffer->capacity()) {
994                handleError(ERROR_BUFFER_TOO_SMALL);
995                mDequeuedInputBuffers.push_back(bufferIx);
996                return false;
997            }
998
999            if (buffer->data() != NULL) {
1000                codecBuffer->setRange(0, buffer->size());
1001                memcpy(codecBuffer->data(), buffer->data(), buffer->size());
1002            } else { // No buffer->data()
1003                //Modular DRM
1004                mediaBuf = (MediaBuffer*)buffer->getMediaBufferBase();
1005                if (mediaBuf != NULL) {
1006                    codecBuffer->setRange(0, mediaBuf->size());
1007                    memcpy(codecBuffer->data(), mediaBuf->data(), mediaBuf->size());
1008
1009                    sp<MetaData> meta_data = mediaBuf->meta_data();
1010                    cryptInfo = NuPlayerDrm::getSampleCryptoInfo(meta_data);
1011
1012                    // since getMediaBuffer() has incremented the refCount
1013                    mediaBuf->release();
1014                } else { // No mediaBuf
1015                    ALOGE("onInputBufferFetched: buffer->data()/mediaBuf are NULL for %p",
1016                            buffer.get());
1017                    handleError(UNKNOWN_ERROR);
1018                    return false;
1019                }
1020            } // buffer->data()
1021        } // needsCopy
1022
1023        status_t err;
1024        AString errorDetailMsg;
1025        if (cryptInfo != NULL) {
1026            err = mCodec->queueSecureInputBuffer(
1027                    bufferIx,
1028                    codecBuffer->offset(),
1029                    cryptInfo->subSamples,
1030                    cryptInfo->numSubSamples,
1031                    cryptInfo->key,
1032                    cryptInfo->iv,
1033                    cryptInfo->mode,
1034                    cryptInfo->pattern,
1035                    timeUs,
1036                    flags,
1037                    &errorDetailMsg);
1038            // synchronous call so done with cryptInfo here
1039            free(cryptInfo);
1040        } else {
1041            err = mCodec->queueInputBuffer(
1042                    bufferIx,
1043                    codecBuffer->offset(),
1044                    codecBuffer->size(),
1045                    timeUs,
1046                    flags,
1047                    &errorDetailMsg);
1048        } // no cryptInfo
1049
1050        if (err != OK) {
1051            ALOGE("onInputBufferFetched: queue%sInputBuffer failed for %s (err=%d, %s)",
1052                    (cryptInfo != NULL ? "Secure" : ""),
1053                    mComponentName.c_str(), err, errorDetailMsg.c_str());
1054            handleError(err);
1055        } else {
1056            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
1057        }
1058
1059    }   // buffer != NULL
1060    return true;
1061}
1062
1063void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
1064    status_t err;
1065    int32_t render;
1066    size_t bufferIx;
1067    int32_t eos;
1068    CHECK(msg->findSize("buffer-ix", &bufferIx));
1069
1070    if (!mIsAudio) {
1071        int64_t timeUs;
1072        sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx];
1073        buffer->meta()->findInt64("timeUs", &timeUs);
1074
1075        if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
1076            mCCDecoder->display(timeUs);
1077        }
1078    }
1079
1080    if (msg->findInt32("render", &render) && render) {
1081        int64_t timestampNs;
1082        CHECK(msg->findInt64("timestampNs", &timestampNs));
1083        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
1084    } else {
1085        mNumOutputFramesDropped += !mIsAudio;
1086        err = mCodec->releaseOutputBuffer(bufferIx);
1087    }
1088    if (err != OK) {
1089        ALOGE("failed to release output buffer for %s (err=%d)",
1090                mComponentName.c_str(), err);
1091        handleError(err);
1092    }
1093    if (msg->findInt32("eos", &eos) && eos
1094            && isDiscontinuityPending()) {
1095        finishHandleDiscontinuity(true /* flushOnTimeChange */);
1096    }
1097}
1098
1099bool NuPlayer::Decoder::isDiscontinuityPending() const {
1100    return mFormatChangePending || mTimeChangePending;
1101}
1102
1103void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
1104    ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
1105            mFormatChangePending, mTimeChangePending, flushOnTimeChange);
1106
1107    // If we have format change, pause and wait to be killed;
1108    // If we have time change only, flush and restart fetching.
1109
1110    if (mFormatChangePending) {
1111        mPaused = true;
1112    } else if (mTimeChangePending) {
1113        if (flushOnTimeChange) {
1114            doFlush(false /* notifyComplete */);
1115            signalResume(false /* notifyComplete */);
1116        }
1117    }
1118
1119    // Notify NuPlayer to either shutdown decoder, or rescan sources
1120    sp<AMessage> msg = mNotify->dup();
1121    msg->setInt32("what", kWhatInputDiscontinuity);
1122    msg->setInt32("formatChange", mFormatChangePending);
1123    msg->post();
1124
1125    mFormatChangePending = false;
1126    mTimeChangePending = false;
1127}
1128
1129bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
1130        const sp<AMessage> &targetFormat) const {
1131    if (targetFormat == NULL) {
1132        return true;
1133    }
1134
1135    AString mime;
1136    if (!targetFormat->findString("mime", &mime)) {
1137        return false;
1138    }
1139
1140    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
1141        // field-by-field comparison
1142        const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
1143        for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
1144            int32_t oldVal, newVal;
1145            if (!mInputFormat->findInt32(keys[i], &oldVal) ||
1146                    !targetFormat->findInt32(keys[i], &newVal) ||
1147                    oldVal != newVal) {
1148                return false;
1149            }
1150        }
1151
1152        sp<ABuffer> oldBuf, newBuf;
1153        if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
1154                targetFormat->findBuffer("csd-0", &newBuf)) {
1155            if (oldBuf->size() != newBuf->size()) {
1156                return false;
1157            }
1158            return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
1159        }
1160    }
1161    return false;
1162}
1163
1164bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
1165    if (mInputFormat == NULL) {
1166        return false;
1167    }
1168
1169    if (targetFormat == NULL) {
1170        return true;
1171    }
1172
1173    AString oldMime, newMime;
1174    if (!mInputFormat->findString("mime", &oldMime)
1175            || !targetFormat->findString("mime", &newMime)
1176            || !(oldMime == newMime)) {
1177        return false;
1178    }
1179
1180    bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
1181    bool seamless;
1182    if (audio) {
1183        seamless = supportsSeamlessAudioFormatChange(targetFormat);
1184    } else {
1185        int32_t isAdaptive;
1186        seamless = (mCodec != NULL &&
1187                mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
1188                isAdaptive);
1189    }
1190
1191    ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
1192    return seamless;
1193}
1194
1195void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
1196    if (format == NULL) {
1197        return;
1198    }
1199    mCSDsForCurrentFormat.clear();
1200    for (int32_t i = 0; ; ++i) {
1201        AString tag = "csd-";
1202        tag.append(i);
1203        sp<ABuffer> buffer;
1204        if (!format->findBuffer(tag.c_str(), &buffer)) {
1205            break;
1206        }
1207        mCSDsForCurrentFormat.push(buffer);
1208    }
1209}
1210
1211void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
1212    if (mResumePending) {
1213        mResumePending = false;
1214
1215        sp<AMessage> notify = mNotify->dup();
1216        notify->setInt32("what", kWhatResumeCompleted);
1217        notify->post();
1218    }
1219}
1220
1221}  // namespace android
1222
1223