NuPlayerDecoder.cpp revision a81c6229638a4db56752dd77a6610e0f0971e877 
1/*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerDecoder"
19#include <utils/Log.h>
20#include <inttypes.h>
21
22#include "NuPlayerCCDecoder.h"
23#include "NuPlayerDecoder.h"
24#include "NuPlayerRenderer.h"
25#include "NuPlayerSource.h"
26
27#include <cutils/properties.h>
28#include <media/ICrypto.h>
29#include <media/stagefright/foundation/ABuffer.h>
30#include <media/stagefright/foundation/ADebug.h>
31#include <media/stagefright/foundation/AMessage.h>
32#include <media/stagefright/MediaBuffer.h>
33#include <media/stagefright/MediaCodec.h>
34#include <media/stagefright/MediaDefs.h>
35#include <media/stagefright/MediaErrors.h>
36
37#include <gui/Surface.h>
38
39#include "avc_utils.h"
40#include "ATSParser.h"
41
42namespace android {
43
44static inline bool getAudioDeepBufferSetting() {
45    return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
46}
47
48NuPlayer::Decoder::Decoder(
49        const sp<AMessage> &notify,
50        const sp<Source> &source,
51        const sp<Renderer> &renderer,
52        const sp<Surface> &surface,
53        const sp<CCDecoder> &ccDecoder)
54    : DecoderBase(notify),
55      mSurface(surface),
56      mSource(source),
57      mRenderer(renderer),
58      mCCDecoder(ccDecoder),
59      mSkipRenderingUntilMediaTimeUs(-1ll),
60      mNumFramesTotal(0ll),
61      mNumInputFramesDropped(0ll),
62      mNumOutputFramesDropped(0ll),
63      mVideoWidth(0),
64      mVideoHeight(0),
65      mIsAudio(true),
66      mIsVideoAVC(false),
67      mIsSecure(false),
68      mFormatChangePending(false),
69      mTimeChangePending(false),
70      mPaused(true),
71      mResumePending(false),
72      mComponentName("decoder") {
73    mCodecLooper = new ALooper;
74    mCodecLooper->setName("NPDecoder-CL");
75    mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
76}
77
78NuPlayer::Decoder::~Decoder() {
79    mCodec->release();
80    releaseAndResetMediaBuffers();
81}
82
83sp<AMessage> NuPlayer::Decoder::getStats() const {
84    mStats->setInt64("frames-total", mNumFramesTotal);
85    mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
86    mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
87    return mStats;
88}
89
90status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
91    if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
92        return BAD_VALUE;
93    }
94
95    sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
96
97    msg->setObject("surface", surface);
98    sp<AMessage> response;
99    status_t err = msg->postAndAwaitResponse(&response);
100    if (err == OK && response != NULL) {
101        CHECK(response->findInt32("err", &err));
102    }
103    return err;
104}
105
106void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
107    ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
108
109    switch (msg->what()) {
110        case kWhatCodecNotify:
111        {
112            int32_t cbID;
113            CHECK(msg->findInt32("callbackID", &cbID));
114
115            ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
116                    mIsAudio ? "audio" : "video", cbID, mPaused);
117
118            if (mPaused) {
119                break;
120            }
121
122            switch (cbID) {
123                case MediaCodec::CB_INPUT_AVAILABLE:
124                {
125                    int32_t index;
126                    CHECK(msg->findInt32("index", &index));
127
128                    handleAnInputBuffer(index);
129                    break;
130                }
131
132                case MediaCodec::CB_OUTPUT_AVAILABLE:
133                {
134                    int32_t index;
135                    size_t offset;
136                    size_t size;
137                    int64_t timeUs;
138                    int32_t flags;
139
140                    CHECK(msg->findInt32("index", &index));
141                    CHECK(msg->findSize("offset", &offset));
142                    CHECK(msg->findSize("size", &size));
143                    CHECK(msg->findInt64("timeUs", &timeUs));
144                    CHECK(msg->findInt32("flags", &flags));
145
146                    handleAnOutputBuffer(index, offset, size, timeUs, flags);
147                    break;
148                }
149
150                case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
151                {
152                    sp<AMessage> format;
153                    CHECK(msg->findMessage("format", &format));
154
155                    handleOutputFormatChange(format);
156                    break;
157                }
158
159                case MediaCodec::CB_ERROR:
160                {
161                    status_t err;
162                    CHECK(msg->findInt32("err", &err));
163                    ALOGE("Decoder (%s) reported error : 0x%x",
164                            mIsAudio ? "audio" : "video", err);
165
166                    handleError(err);
167                    break;
168                }
169
170                default:
171                {
172                    TRESPASS();
173                    break;
174                }
175            }
176
177            break;
178        }
179
180        case kWhatRenderBuffer:
181        {
182            if (!isStaleReply(msg)) {
183                onRenderBuffer(msg);
184            }
185            break;
186        }
187
188        case kWhatSetVideoSurface:
189        {
190            sp<AReplyToken> replyID;
191            CHECK(msg->senderAwaitsResponse(&replyID));
192
193            sp<RefBase> obj;
194            CHECK(msg->findObject("surface", &obj));
195            sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
196            int32_t err = INVALID_OPERATION;
197            // NOTE: in practice mSurface is always non-null, but checking here for completeness
198            if (mCodec != NULL && mSurface != NULL) {
199                // TODO: once AwesomePlayer is removed, remove this automatic connecting
200                // to the surface by MediaPlayerService.
201                //
202                // at this point MediaPlayerService::client has already connected to the
203                // surface, which MediaCodec does not expect
204                err = native_window_api_disconnect(surface.get(), NATIVE_WINDOW_API_MEDIA);
205                if (err == OK) {
206                    err = mCodec->setSurface(surface);
207                    ALOGI_IF(err, "codec setSurface returned: %d", err);
208                    if (err == OK) {
209                        // reconnect to the old surface as MPS::Client will expect to
210                        // be able to disconnect from it.
211                        (void)native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
212                        mSurface = surface;
213                    }
214                }
215                if (err != OK) {
216                    // reconnect to the new surface on error as MPS::Client will expect to
217                    // be able to disconnect from it.
218                    (void)native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA);
219                }
220            }
221
222            sp<AMessage> response = new AMessage;
223            response->setInt32("err", err);
224            response->postReply(replyID);
225            break;
226        }
227
228        default:
229            DecoderBase::onMessageReceived(msg);
230            break;
231    }
232}
233
234void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
235    CHECK(mCodec == NULL);
236
237    mFormatChangePending = false;
238    mTimeChangePending = false;
239
240    ++mBufferGeneration;
241
242    AString mime;
243    CHECK(format->findString("mime", &mime));
244
245    mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
246    mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
247
248    mComponentName = mime;
249    mComponentName.append(" decoder");
250    ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
251
252    mCodec = MediaCodec::CreateByType(mCodecLooper, mime.c_str(), false /* encoder */);
253    int32_t secure = 0;
254    if (format->findInt32("secure", &secure) && secure != 0) {
255        if (mCodec != NULL) {
256            mCodec->getName(&mComponentName);
257            mComponentName.append(".secure");
258            mCodec->release();
259            ALOGI("[%s] creating", mComponentName.c_str());
260            mCodec = MediaCodec::CreateByComponentName(
261                    mCodecLooper, mComponentName.c_str());
262        }
263    }
264    if (mCodec == NULL) {
265        ALOGE("Failed to create %s%s decoder",
266                (secure ? "secure " : ""), mime.c_str());
267        handleError(UNKNOWN_ERROR);
268        return;
269    }
270    mIsSecure = secure;
271
272    mCodec->getName(&mComponentName);
273
274    status_t err;
275    if (mSurface != NULL) {
276        // disconnect from surface as MediaCodec will reconnect
277        err = native_window_api_disconnect(
278                mSurface.get(), NATIVE_WINDOW_API_MEDIA);
279        // We treat this as a warning, as this is a preparatory step.
280        // Codec will try to connect to the surface, which is where
281        // any error signaling will occur.
282        ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
283    }
284    err = mCodec->configure(
285            format, mSurface, NULL /* crypto */, 0 /* flags */);
286    if (err != OK) {
287        ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
288        mCodec->release();
289        mCodec.clear();
290        handleError(err);
291        return;
292    }
293    rememberCodecSpecificData(format);
294
295    // the following should work in configured state
296    CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
297    CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
298
299    mStats->setString("mime", mime.c_str());
300    mStats->setString("component-name", mComponentName.c_str());
301
302    if (!mIsAudio) {
303        int32_t width, height;
304        if (mOutputFormat->findInt32("width", &width)
305                && mOutputFormat->findInt32("height", &height)) {
306            mStats->setInt32("width", width);
307            mStats->setInt32("height", height);
308        }
309    }
310
311    sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
312    mCodec->setCallback(reply);
313
314    err = mCodec->start();
315    if (err != OK) {
316        ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
317        mCodec->release();
318        mCodec.clear();
319        handleError(err);
320        return;
321    }
322
323    releaseAndResetMediaBuffers();
324
325    mPaused = false;
326    mResumePending = false;
327}
328
329void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) {
330    if (mCodec == NULL) {
331        ALOGW("onSetParameters called before codec is created.");
332        return;
333    }
334    mCodec->setParameters(params);
335}
336
337void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
338    bool hadNoRenderer = (mRenderer == NULL);
339    mRenderer = renderer;
340    if (hadNoRenderer && mRenderer != NULL) {
341        // this means that the widevine legacy source is ready
342        onRequestInputBuffers();
343    }
344}
345
346void NuPlayer::Decoder::onGetInputBuffers(
347        Vector<sp<ABuffer> > *dstBuffers) {
348    CHECK_EQ((status_t)OK, mCodec->getWidevineLegacyBuffers(dstBuffers));
349}
350
351void NuPlayer::Decoder::onResume(bool notifyComplete) {
352    mPaused = false;
353
354    if (notifyComplete) {
355        mResumePending = true;
356    }
357    mCodec->start();
358}
359
360void NuPlayer::Decoder::doFlush(bool notifyComplete) {
361    if (mCCDecoder != NULL) {
362        mCCDecoder->flush();
363    }
364
365    if (mRenderer != NULL) {
366        mRenderer->flush(mIsAudio, notifyComplete);
367        mRenderer->signalTimeDiscontinuity();
368    }
369
370    status_t err = OK;
371    if (mCodec != NULL) {
372        err = mCodec->flush();
373        mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
374        ++mBufferGeneration;
375    }
376
377    if (err != OK) {
378        ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
379        handleError(err);
380        // finish with posting kWhatFlushCompleted.
381        // we attempt to release the buffers even if flush fails.
382    }
383    releaseAndResetMediaBuffers();
384    mPaused = true;
385}
386
387
388void NuPlayer::Decoder::onFlush() {
389    doFlush(true);
390
391    if (isDiscontinuityPending()) {
392        // This could happen if the client starts seeking/shutdown
393        // after we queued an EOS for discontinuities.
394        // We can consider discontinuity handled.
395        finishHandleDiscontinuity(false /* flushOnTimeChange */);
396    }
397
398    sp<AMessage> notify = mNotify->dup();
399    notify->setInt32("what", kWhatFlushCompleted);
400    notify->post();
401}
402
403void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
404    status_t err = OK;
405
406    // if there is a pending resume request, notify complete now
407    notifyResumeCompleteIfNecessary();
408
409    if (mCodec != NULL) {
410        err = mCodec->release();
411        mCodec = NULL;
412        ++mBufferGeneration;
413
414        if (mSurface != NULL) {
415            // reconnect to surface as MediaCodec disconnected from it
416            status_t error =
417                    native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
418            ALOGW_IF(error != NO_ERROR,
419                    "[%s] failed to connect to native window, error=%d",
420                    mComponentName.c_str(), error);
421        }
422        mComponentName = "decoder";
423    }
424
425    releaseAndResetMediaBuffers();
426
427    if (err != OK) {
428        ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
429        handleError(err);
430        // finish with posting kWhatShutdownCompleted.
431    }
432
433    if (notifyComplete) {
434        sp<AMessage> notify = mNotify->dup();
435        notify->setInt32("what", kWhatShutdownCompleted);
436        notify->post();
437        mPaused = true;
438    }
439}
440
441/*
442 * returns true if we should request more data
443 */
444bool NuPlayer::Decoder::doRequestBuffers() {
445    // mRenderer is only NULL if we have a legacy widevine source that
446    // is not yet ready. In this case we must not fetch input.
447    if (isDiscontinuityPending() || mRenderer == NULL) {
448        return false;
449    }
450    status_t err = OK;
451    while (err == OK && !mDequeuedInputBuffers.empty()) {
452        size_t bufferIx = *mDequeuedInputBuffers.begin();
453        sp<AMessage> msg = new AMessage();
454        msg->setSize("buffer-ix", bufferIx);
455        err = fetchInputData(msg);
456        if (err != OK && err != ERROR_END_OF_STREAM) {
457            // if EOS, need to queue EOS buffer
458            break;
459        }
460        mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
461
462        if (!mPendingInputMessages.empty()
463                || !onInputBufferFetched(msg)) {
464            mPendingInputMessages.push_back(msg);
465        }
466    }
467
468    return err == -EWOULDBLOCK
469            && mSource->feedMoreTSData() == OK;
470}
471
472void NuPlayer::Decoder::handleError(int32_t err)
473{
474    // We cannot immediately release the codec due to buffers still outstanding
475    // in the renderer.  We signal to the player the error so it can shutdown/release the
476    // decoder after flushing and increment the generation to discard unnecessary messages.
477
478    ++mBufferGeneration;
479
480    sp<AMessage> notify = mNotify->dup();
481    notify->setInt32("what", kWhatError);
482    notify->setInt32("err", err);
483    notify->post();
484}
485
486bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
487    if (isDiscontinuityPending()) {
488        return false;
489    }
490
491    sp<ABuffer> buffer;
492    mCodec->getInputBuffer(index, &buffer);
493
494    if (buffer == NULL) {
495        handleError(UNKNOWN_ERROR);
496        return false;
497    }
498
499    if (index >= mInputBuffers.size()) {
500        for (size_t i = mInputBuffers.size(); i <= index; ++i) {
501            mInputBuffers.add();
502            mMediaBuffers.add();
503            mInputBufferIsDequeued.add();
504            mMediaBuffers.editItemAt(i) = NULL;
505            mInputBufferIsDequeued.editItemAt(i) = false;
506        }
507    }
508    mInputBuffers.editItemAt(index) = buffer;
509
510    //CHECK_LT(bufferIx, mInputBuffers.size());
511
512    if (mMediaBuffers[index] != NULL) {
513        mMediaBuffers[index]->release();
514        mMediaBuffers.editItemAt(index) = NULL;
515    }
516    mInputBufferIsDequeued.editItemAt(index) = true;
517
518    if (!mCSDsToSubmit.isEmpty()) {
519        sp<AMessage> msg = new AMessage();
520        msg->setSize("buffer-ix", index);
521
522        sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
523        ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
524        msg->setBuffer("buffer", buffer);
525        mCSDsToSubmit.removeAt(0);
526        CHECK(onInputBufferFetched(msg));
527        return true;
528    }
529
530    while (!mPendingInputMessages.empty()) {
531        sp<AMessage> msg = *mPendingInputMessages.begin();
532        if (!onInputBufferFetched(msg)) {
533            break;
534        }
535        mPendingInputMessages.erase(mPendingInputMessages.begin());
536    }
537
538    if (!mInputBufferIsDequeued.editItemAt(index)) {
539        return true;
540    }
541
542    mDequeuedInputBuffers.push_back(index);
543
544    onRequestInputBuffers();
545    return true;
546}
547
548bool NuPlayer::Decoder::handleAnOutputBuffer(
549        size_t index,
550        size_t offset,
551        size_t size,
552        int64_t timeUs,
553        int32_t flags) {
554//    CHECK_LT(bufferIx, mOutputBuffers.size());
555    sp<ABuffer> buffer;
556    mCodec->getOutputBuffer(index, &buffer);
557
558    if (index >= mOutputBuffers.size()) {
559        for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
560            mOutputBuffers.add();
561        }
562    }
563
564    mOutputBuffers.editItemAt(index) = buffer;
565
566    buffer->setRange(offset, size);
567    buffer->meta()->clear();
568    buffer->meta()->setInt64("timeUs", timeUs);
569
570    bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
571    // we do not expect CODECCONFIG or SYNCFRAME for decoder
572
573    sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
574    reply->setSize("buffer-ix", index);
575    reply->setInt32("generation", mBufferGeneration);
576
577    if (eos) {
578        ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
579
580        buffer->meta()->setInt32("eos", true);
581        reply->setInt32("eos", true);
582    } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
583        if (timeUs < mSkipRenderingUntilMediaTimeUs) {
584            ALOGV("[%s] dropping buffer at time %lld as requested.",
585                     mComponentName.c_str(), (long long)timeUs);
586
587            reply->post();
588            return true;
589        }
590
591        mSkipRenderingUntilMediaTimeUs = -1;
592    }
593
594    mNumFramesTotal += !mIsAudio;
595
596    // wait until 1st frame comes out to signal resume complete
597    notifyResumeCompleteIfNecessary();
598
599    if (mRenderer != NULL) {
600        // send the buffer to renderer.
601        mRenderer->queueBuffer(mIsAudio, buffer, reply);
602        if (eos && !isDiscontinuityPending()) {
603            mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
604        }
605    }
606
607    return true;
608}
609
610void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
611    if (!mIsAudio) {
612        int32_t width, height;
613        if (format->findInt32("width", &width)
614                && format->findInt32("height", &height)) {
615            mStats->setInt32("width", width);
616            mStats->setInt32("height", height);
617        }
618        sp<AMessage> notify = mNotify->dup();
619        notify->setInt32("what", kWhatVideoSizeChanged);
620        notify->setMessage("format", format);
621        notify->post();
622    } else if (mRenderer != NULL) {
623        uint32_t flags;
624        int64_t durationUs;
625        bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
626        if (getAudioDeepBufferSetting() // override regardless of source duration
627                || (!hasVideo
628                        && mSource->getDuration(&durationUs) == OK
629                        && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
630            flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
631        } else {
632            flags = AUDIO_OUTPUT_FLAG_NONE;
633        }
634
635        mRenderer->openAudioSink(
636                format, false /* offloadOnly */, hasVideo, flags, NULL /* isOffloaed */);
637    }
638}
639
640void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
641    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
642        if (mMediaBuffers[i] != NULL) {
643            mMediaBuffers[i]->release();
644            mMediaBuffers.editItemAt(i) = NULL;
645        }
646    }
647    mMediaBuffers.resize(mInputBuffers.size());
648    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
649        mMediaBuffers.editItemAt(i) = NULL;
650    }
651    mInputBufferIsDequeued.clear();
652    mInputBufferIsDequeued.resize(mInputBuffers.size());
653    for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
654        mInputBufferIsDequeued.editItemAt(i) = false;
655    }
656
657    mPendingInputMessages.clear();
658    mDequeuedInputBuffers.clear();
659    mSkipRenderingUntilMediaTimeUs = -1;
660}
661
662void NuPlayer::Decoder::requestCodecNotification() {
663    if (mCodec != NULL) {
664        sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
665        reply->setInt32("generation", mBufferGeneration);
666        mCodec->requestActivityNotification(reply);
667    }
668}
669
670bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
671    int32_t generation;
672    CHECK(msg->findInt32("generation", &generation));
673    return generation != mBufferGeneration;
674}
675
676status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
677    sp<ABuffer> accessUnit;
678    bool dropAccessUnit;
679    do {
680        status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
681
682        if (err == -EWOULDBLOCK) {
683            return err;
684        } else if (err != OK) {
685            if (err == INFO_DISCONTINUITY) {
686                int32_t type;
687                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
688
689                bool formatChange =
690                    (mIsAudio &&
691                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
692                    || (!mIsAudio &&
693                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
694
695                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
696
697                ALOGI("%s discontinuity (format=%d, time=%d)",
698                        mIsAudio ? "audio" : "video", formatChange, timeChange);
699
700                bool seamlessFormatChange = false;
701                sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
702                if (formatChange) {
703                    seamlessFormatChange =
704                        supportsSeamlessFormatChange(newFormat);
705                    // treat seamless format change separately
706                    formatChange = !seamlessFormatChange;
707                }
708
709                // For format or time change, return EOS to queue EOS input,
710                // then wait for EOS on output.
711                if (formatChange /* not seamless */) {
712                    mFormatChangePending = true;
713                    err = ERROR_END_OF_STREAM;
714                } else if (timeChange) {
715                    rememberCodecSpecificData(newFormat);
716                    mTimeChangePending = true;
717                    err = ERROR_END_OF_STREAM;
718                } else if (seamlessFormatChange) {
719                    // reuse existing decoder and don't flush
720                    rememberCodecSpecificData(newFormat);
721                    continue;
722                } else {
723                    // This stream is unaffected by the discontinuity
724                    return -EWOULDBLOCK;
725                }
726            }
727
728            // reply should only be returned without a buffer set
729            // when there is an error (including EOS)
730            CHECK(err != OK);
731
732            reply->setInt32("err", err);
733            return ERROR_END_OF_STREAM;
734        }
735
736        dropAccessUnit = false;
737        if (!mIsAudio
738                && !mIsSecure
739                && mRenderer->getVideoLateByUs() > 100000ll
740                && mIsVideoAVC
741                && !IsAVCReferenceFrame(accessUnit)) {
742            dropAccessUnit = true;
743            ++mNumInputFramesDropped;
744        }
745    } while (dropAccessUnit);
746
747    // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
748#if 0
749    int64_t mediaTimeUs;
750    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
751    ALOGV("[%s] feeding input buffer at media time %.3f",
752         mIsAudio ? "audio" : "video",
753         mediaTimeUs / 1E6);
754#endif
755
756    if (mCCDecoder != NULL) {
757        mCCDecoder->decode(accessUnit);
758    }
759
760    reply->setBuffer("buffer", accessUnit);
761
762    return OK;
763}
764
765bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
766    size_t bufferIx;
767    CHECK(msg->findSize("buffer-ix", &bufferIx));
768    CHECK_LT(bufferIx, mInputBuffers.size());
769    sp<ABuffer> codecBuffer = mInputBuffers[bufferIx];
770
771    sp<ABuffer> buffer;
772    bool hasBuffer = msg->findBuffer("buffer", &buffer);
773
774    // handle widevine classic source - that fills an arbitrary input buffer
775    MediaBuffer *mediaBuffer = NULL;
776    if (hasBuffer) {
777        mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase());
778        if (mediaBuffer != NULL) {
779            // likely filled another buffer than we requested: adjust buffer index
780            size_t ix;
781            for (ix = 0; ix < mInputBuffers.size(); ix++) {
782                const sp<ABuffer> &buf = mInputBuffers[ix];
783                if (buf->data() == mediaBuffer->data()) {
784                    // all input buffers are dequeued on start, hence the check
785                    if (!mInputBufferIsDequeued[ix]) {
786                        ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu",
787                                mComponentName.c_str(), ix, bufferIx);
788                        mediaBuffer->release();
789                        return false;
790                    }
791
792                    // TRICKY: need buffer for the metadata, so instead, set
793                    // codecBuffer to the same (though incorrect) buffer to
794                    // avoid a memcpy into the codecBuffer
795                    codecBuffer = buffer;
796                    codecBuffer->setRange(
797                            mediaBuffer->range_offset(),
798                            mediaBuffer->range_length());
799                    bufferIx = ix;
800                    break;
801                }
802            }
803            CHECK(ix < mInputBuffers.size());
804        }
805    }
806
807    if (buffer == NULL /* includes !hasBuffer */) {
808        int32_t streamErr = ERROR_END_OF_STREAM;
809        CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
810
811        CHECK(streamErr != OK);
812
813        // attempt to queue EOS
814        status_t err = mCodec->queueInputBuffer(
815                bufferIx,
816                0,
817                0,
818                0,
819                MediaCodec::BUFFER_FLAG_EOS);
820        if (err == OK) {
821            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
822        } else if (streamErr == ERROR_END_OF_STREAM) {
823            streamErr = err;
824            // err will not be ERROR_END_OF_STREAM
825        }
826
827        if (streamErr != ERROR_END_OF_STREAM) {
828            ALOGE("Stream error for %s (err=%d), EOS %s queued",
829                    mComponentName.c_str(),
830                    streamErr,
831                    err == OK ? "successfully" : "unsuccessfully");
832            handleError(streamErr);
833        }
834    } else {
835        sp<AMessage> extra;
836        if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
837            int64_t resumeAtMediaTimeUs;
838            if (extra->findInt64(
839                        "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
840                ALOGI("[%s] suppressing rendering until %lld us",
841                        mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
842                mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
843            }
844        }
845
846        int64_t timeUs = 0;
847        uint32_t flags = 0;
848        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
849
850        int32_t eos, csd;
851        // we do not expect SYNCFRAME for decoder
852        if (buffer->meta()->findInt32("eos", &eos) && eos) {
853            flags |= MediaCodec::BUFFER_FLAG_EOS;
854        } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
855            flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
856        }
857
858        // copy into codec buffer
859        if (buffer != codecBuffer) {
860            CHECK_LE(buffer->size(), codecBuffer->capacity());
861            codecBuffer->setRange(0, buffer->size());
862            memcpy(codecBuffer->data(), buffer->data(), buffer->size());
863        }
864
865        status_t err = mCodec->queueInputBuffer(
866                        bufferIx,
867                        codecBuffer->offset(),
868                        codecBuffer->size(),
869                        timeUs,
870                        flags);
871        if (err != OK) {
872            if (mediaBuffer != NULL) {
873                mediaBuffer->release();
874            }
875            ALOGE("Failed to queue input buffer for %s (err=%d)",
876                    mComponentName.c_str(), err);
877            handleError(err);
878        } else {
879            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
880            if (mediaBuffer != NULL) {
881                CHECK(mMediaBuffers[bufferIx] == NULL);
882                mMediaBuffers.editItemAt(bufferIx) = mediaBuffer;
883            }
884        }
885    }
886    return true;
887}
888
889void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
890    status_t err;
891    int32_t render;
892    size_t bufferIx;
893    int32_t eos;
894    CHECK(msg->findSize("buffer-ix", &bufferIx));
895
896    if (!mIsAudio) {
897        int64_t timeUs;
898        sp<ABuffer> buffer = mOutputBuffers[bufferIx];
899        buffer->meta()->findInt64("timeUs", &timeUs);
900
901        if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
902            mCCDecoder->display(timeUs);
903        }
904    }
905
906    if (msg->findInt32("render", &render) && render) {
907        int64_t timestampNs;
908        CHECK(msg->findInt64("timestampNs", &timestampNs));
909        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
910    } else {
911        mNumOutputFramesDropped += !mIsAudio;
912        err = mCodec->releaseOutputBuffer(bufferIx);
913    }
914    if (err != OK) {
915        ALOGE("failed to release output buffer for %s (err=%d)",
916                mComponentName.c_str(), err);
917        handleError(err);
918    }
919    if (msg->findInt32("eos", &eos) && eos
920            && isDiscontinuityPending()) {
921        finishHandleDiscontinuity(true /* flushOnTimeChange */);
922    }
923}
924
925bool NuPlayer::Decoder::isDiscontinuityPending() const {
926    return mFormatChangePending || mTimeChangePending;
927}
928
929void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
930    ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
931            mFormatChangePending, mTimeChangePending, flushOnTimeChange);
932
933    // If we have format change, pause and wait to be killed;
934    // If we have time change only, flush and restart fetching.
935
936    if (mFormatChangePending) {
937        mPaused = true;
938    } else if (mTimeChangePending) {
939        if (flushOnTimeChange) {
940            doFlush(false /* notifyComplete */);
941            signalResume(false /* notifyComplete */);
942        }
943    }
944
945    // Notify NuPlayer to either shutdown decoder, or rescan sources
946    sp<AMessage> msg = mNotify->dup();
947    msg->setInt32("what", kWhatInputDiscontinuity);
948    msg->setInt32("formatChange", mFormatChangePending);
949    msg->post();
950
951    mFormatChangePending = false;
952    mTimeChangePending = false;
953}
954
955bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
956        const sp<AMessage> &targetFormat) const {
957    if (targetFormat == NULL) {
958        return true;
959    }
960
961    AString mime;
962    if (!targetFormat->findString("mime", &mime)) {
963        return false;
964    }
965
966    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
967        // field-by-field comparison
968        const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
969        for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
970            int32_t oldVal, newVal;
971            if (!mInputFormat->findInt32(keys[i], &oldVal) ||
972                    !targetFormat->findInt32(keys[i], &newVal) ||
973                    oldVal != newVal) {
974                return false;
975            }
976        }
977
978        sp<ABuffer> oldBuf, newBuf;
979        if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
980                targetFormat->findBuffer("csd-0", &newBuf)) {
981            if (oldBuf->size() != newBuf->size()) {
982                return false;
983            }
984            return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
985        }
986    }
987    return false;
988}
989
990bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
991    if (mInputFormat == NULL) {
992        return false;
993    }
994
995    if (targetFormat == NULL) {
996        return true;
997    }
998
999    AString oldMime, newMime;
1000    if (!mInputFormat->findString("mime", &oldMime)
1001            || !targetFormat->findString("mime", &newMime)
1002            || !(oldMime == newMime)) {
1003        return false;
1004    }
1005
1006    bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
1007    bool seamless;
1008    if (audio) {
1009        seamless = supportsSeamlessAudioFormatChange(targetFormat);
1010    } else {
1011        int32_t isAdaptive;
1012        seamless = (mCodec != NULL &&
1013                mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
1014                isAdaptive);
1015    }
1016
1017    ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
1018    return seamless;
1019}
1020
1021void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
1022    if (format == NULL) {
1023        return;
1024    }
1025    mCSDsForCurrentFormat.clear();
1026    for (int32_t i = 0; ; ++i) {
1027        AString tag = "csd-";
1028        tag.append(i);
1029        sp<ABuffer> buffer;
1030        if (!format->findBuffer(tag.c_str(), &buffer)) {
1031            break;
1032        }
1033        mCSDsForCurrentFormat.push(buffer);
1034    }
1035}
1036
1037void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
1038    if (mResumePending) {
1039        mResumePending = false;
1040
1041        sp<AMessage> notify = mNotify->dup();
1042        notify->setInt32("what", kWhatResumeCompleted);
1043        notify->post();
1044    }
1045}
1046
1047}  // namespace android
1048
1049