NuPlayerDecoder.cpp revision 3bc667014875aba35102941b3997d242c303aa0d 
1/*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerDecoder"
19#include <utils/Log.h>
20#include <inttypes.h>
21
22#include "NuPlayerCCDecoder.h"
23#include "NuPlayerDecoder.h"
24#include "NuPlayerRenderer.h"
25#include "NuPlayerSource.h"
26
27#include <cutils/properties.h>
28#include <media/ICrypto.h>
29#include <media/stagefright/foundation/ABuffer.h>
30#include <media/stagefright/foundation/ADebug.h>
31#include <media/stagefright/foundation/AMessage.h>
32#include <media/stagefright/MediaBuffer.h>
33#include <media/stagefright/MediaCodec.h>
34#include <media/stagefright/MediaDefs.h>
35#include <media/stagefright/MediaErrors.h>
36
37#include <gui/Surface.h>
38
39#include "avc_utils.h"
40#include "ATSParser.h"
41
42namespace android {
43
44static inline bool getAudioDeepBufferSetting() {
45    return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
46}
47
48NuPlayer::Decoder::Decoder(
49        const sp<AMessage> &notify,
50        const sp<Source> &source,
51        pid_t pid,
52        const sp<Renderer> &renderer,
53        const sp<Surface> &surface,
54        const sp<CCDecoder> &ccDecoder)
55    : DecoderBase(notify),
56      mSurface(surface),
57      mSource(source),
58      mRenderer(renderer),
59      mCCDecoder(ccDecoder),
60      mPid(pid),
61      mSkipRenderingUntilMediaTimeUs(-1ll),
62      mNumFramesTotal(0ll),
63      mNumInputFramesDropped(0ll),
64      mNumOutputFramesDropped(0ll),
65      mVideoWidth(0),
66      mVideoHeight(0),
67      mIsAudio(true),
68      mIsVideoAVC(false),
69      mIsSecure(false),
70      mFormatChangePending(false),
71      mTimeChangePending(false),
72      mResumePending(false),
73      mComponentName("decoder") {
74    mCodecLooper = new ALooper;
75    mCodecLooper->setName("NPDecoder-CL");
76    mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
77}
78
79NuPlayer::Decoder::~Decoder() {
80    mCodec->release();
81    releaseAndResetMediaBuffers();
82}
83
84sp<AMessage> NuPlayer::Decoder::getStats() const {
85    mStats->setInt64("frames-total", mNumFramesTotal);
86    mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
87    mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
88    return mStats;
89}
90
91status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
92    if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
93        return BAD_VALUE;
94    }
95
96    sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
97
98    msg->setObject("surface", surface);
99    sp<AMessage> response;
100    status_t err = msg->postAndAwaitResponse(&response);
101    if (err == OK && response != NULL) {
102        CHECK(response->findInt32("err", &err));
103    }
104    return err;
105}
106
107void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
108    ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
109
110    switch (msg->what()) {
111        case kWhatCodecNotify:
112        {
113            int32_t cbID;
114            CHECK(msg->findInt32("callbackID", &cbID));
115
116            ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
117                    mIsAudio ? "audio" : "video", cbID, mPaused);
118
119            if (mPaused) {
120                break;
121            }
122
123            switch (cbID) {
124                case MediaCodec::CB_INPUT_AVAILABLE:
125                {
126                    int32_t index;
127                    CHECK(msg->findInt32("index", &index));
128
129                    handleAnInputBuffer(index);
130                    break;
131                }
132
133                case MediaCodec::CB_OUTPUT_AVAILABLE:
134                {
135                    int32_t index;
136                    size_t offset;
137                    size_t size;
138                    int64_t timeUs;
139                    int32_t flags;
140
141                    CHECK(msg->findInt32("index", &index));
142                    CHECK(msg->findSize("offset", &offset));
143                    CHECK(msg->findSize("size", &size));
144                    CHECK(msg->findInt64("timeUs", &timeUs));
145                    CHECK(msg->findInt32("flags", &flags));
146
147                    handleAnOutputBuffer(index, offset, size, timeUs, flags);
148                    break;
149                }
150
151                case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
152                {
153                    sp<AMessage> format;
154                    CHECK(msg->findMessage("format", &format));
155
156                    handleOutputFormatChange(format);
157                    break;
158                }
159
160                case MediaCodec::CB_ERROR:
161                {
162                    status_t err;
163                    CHECK(msg->findInt32("err", &err));
164                    ALOGE("Decoder (%s) reported error : 0x%x",
165                            mIsAudio ? "audio" : "video", err);
166
167                    handleError(err);
168                    break;
169                }
170
171                default:
172                {
173                    TRESPASS();
174                    break;
175                }
176            }
177
178            break;
179        }
180
181        case kWhatRenderBuffer:
182        {
183            if (!isStaleReply(msg)) {
184                onRenderBuffer(msg);
185            }
186            break;
187        }
188
189        case kWhatSetVideoSurface:
190        {
191            sp<AReplyToken> replyID;
192            CHECK(msg->senderAwaitsResponse(&replyID));
193
194            sp<RefBase> obj;
195            CHECK(msg->findObject("surface", &obj));
196            sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
197            int32_t err = INVALID_OPERATION;
198            // NOTE: in practice mSurface is always non-null, but checking here for completeness
199            if (mCodec != NULL && mSurface != NULL) {
200                // TODO: once AwesomePlayer is removed, remove this automatic connecting
201                // to the surface by MediaPlayerService.
202                //
203                // at this point MediaPlayerService::client has already connected to the
204                // surface, which MediaCodec does not expect
205                err = native_window_api_disconnect(surface.get(), NATIVE_WINDOW_API_MEDIA);
206                if (err == OK) {
207                    err = mCodec->setSurface(surface);
208                    ALOGI_IF(err, "codec setSurface returned: %d", err);
209                    if (err == OK) {
210                        // reconnect to the old surface as MPS::Client will expect to
211                        // be able to disconnect from it.
212                        (void)native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
213                        mSurface = surface;
214                    }
215                }
216                if (err != OK) {
217                    // reconnect to the new surface on error as MPS::Client will expect to
218                    // be able to disconnect from it.
219                    (void)native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA);
220                }
221            }
222
223            sp<AMessage> response = new AMessage;
224            response->setInt32("err", err);
225            response->postReply(replyID);
226            break;
227        }
228
229        default:
230            DecoderBase::onMessageReceived(msg);
231            break;
232    }
233}
234
235void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
236    CHECK(mCodec == NULL);
237
238    mFormatChangePending = false;
239    mTimeChangePending = false;
240
241    ++mBufferGeneration;
242
243    AString mime;
244    CHECK(format->findString("mime", &mime));
245
246    mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
247    mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
248
249    mComponentName = mime;
250    mComponentName.append(" decoder");
251    ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
252
253    mCodec = MediaCodec::CreateByType(
254            mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid);
255    int32_t secure = 0;
256    if (format->findInt32("secure", &secure) && secure != 0) {
257        if (mCodec != NULL) {
258            mCodec->getName(&mComponentName);
259            mComponentName.append(".secure");
260            mCodec->release();
261            ALOGI("[%s] creating", mComponentName.c_str());
262            mCodec = MediaCodec::CreateByComponentName(
263                    mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid);
264        }
265    }
266    if (mCodec == NULL) {
267        ALOGE("Failed to create %s%s decoder",
268                (secure ? "secure " : ""), mime.c_str());
269        handleError(UNKNOWN_ERROR);
270        return;
271    }
272    mIsSecure = secure;
273
274    mCodec->getName(&mComponentName);
275
276    status_t err;
277    if (mSurface != NULL) {
278        // disconnect from surface as MediaCodec will reconnect
279        err = native_window_api_disconnect(
280                mSurface.get(), NATIVE_WINDOW_API_MEDIA);
281        // We treat this as a warning, as this is a preparatory step.
282        // Codec will try to connect to the surface, which is where
283        // any error signaling will occur.
284        ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
285    }
286    err = mCodec->configure(
287            format, mSurface, NULL /* crypto */, 0 /* flags */);
288    if (err != OK) {
289        ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
290        mCodec->release();
291        mCodec.clear();
292        handleError(err);
293        return;
294    }
295    rememberCodecSpecificData(format);
296
297    // the following should work in configured state
298    CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
299    CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
300
301    mStats->setString("mime", mime.c_str());
302    mStats->setString("component-name", mComponentName.c_str());
303
304    if (!mIsAudio) {
305        int32_t width, height;
306        if (mOutputFormat->findInt32("width", &width)
307                && mOutputFormat->findInt32("height", &height)) {
308            mStats->setInt32("width", width);
309            mStats->setInt32("height", height);
310        }
311    }
312
313    sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
314    mCodec->setCallback(reply);
315
316    err = mCodec->start();
317    if (err != OK) {
318        ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
319        mCodec->release();
320        mCodec.clear();
321        handleError(err);
322        return;
323    }
324
325    releaseAndResetMediaBuffers();
326
327    mPaused = false;
328    mResumePending = false;
329}
330
331void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) {
332    if (mCodec == NULL) {
333        ALOGW("onSetParameters called before codec is created.");
334        return;
335    }
336    mCodec->setParameters(params);
337}
338
339void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
340    bool hadNoRenderer = (mRenderer == NULL);
341    mRenderer = renderer;
342    if (hadNoRenderer && mRenderer != NULL) {
343        // this means that the widevine legacy source is ready
344        onRequestInputBuffers();
345    }
346}
347
348void NuPlayer::Decoder::onGetInputBuffers(
349        Vector<sp<ABuffer> > *dstBuffers) {
350    CHECK_EQ((status_t)OK, mCodec->getWidevineLegacyBuffers(dstBuffers));
351}
352
353void NuPlayer::Decoder::onResume(bool notifyComplete) {
354    mPaused = false;
355
356    if (notifyComplete) {
357        mResumePending = true;
358    }
359    mCodec->start();
360}
361
362void NuPlayer::Decoder::doFlush(bool notifyComplete) {
363    if (mCCDecoder != NULL) {
364        mCCDecoder->flush();
365    }
366
367    if (mRenderer != NULL) {
368        mRenderer->flush(mIsAudio, notifyComplete);
369        mRenderer->signalTimeDiscontinuity();
370    }
371
372    status_t err = OK;
373    if (mCodec != NULL) {
374        err = mCodec->flush();
375        mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
376        ++mBufferGeneration;
377    }
378
379    if (err != OK) {
380        ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
381        handleError(err);
382        // finish with posting kWhatFlushCompleted.
383        // we attempt to release the buffers even if flush fails.
384    }
385    releaseAndResetMediaBuffers();
386    mPaused = true;
387}
388
389
390void NuPlayer::Decoder::onFlush() {
391    doFlush(true);
392
393    if (isDiscontinuityPending()) {
394        // This could happen if the client starts seeking/shutdown
395        // after we queued an EOS for discontinuities.
396        // We can consider discontinuity handled.
397        finishHandleDiscontinuity(false /* flushOnTimeChange */);
398    }
399
400    sp<AMessage> notify = mNotify->dup();
401    notify->setInt32("what", kWhatFlushCompleted);
402    notify->post();
403}
404
405void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
406    status_t err = OK;
407
408    // if there is a pending resume request, notify complete now
409    notifyResumeCompleteIfNecessary();
410
411    if (mCodec != NULL) {
412        err = mCodec->release();
413        mCodec = NULL;
414        ++mBufferGeneration;
415
416        if (mSurface != NULL) {
417            // reconnect to surface as MediaCodec disconnected from it
418            status_t error =
419                    native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
420            ALOGW_IF(error != NO_ERROR,
421                    "[%s] failed to connect to native window, error=%d",
422                    mComponentName.c_str(), error);
423        }
424        mComponentName = "decoder";
425    }
426
427    releaseAndResetMediaBuffers();
428
429    if (err != OK) {
430        ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
431        handleError(err);
432        // finish with posting kWhatShutdownCompleted.
433    }
434
435    if (notifyComplete) {
436        sp<AMessage> notify = mNotify->dup();
437        notify->setInt32("what", kWhatShutdownCompleted);
438        notify->post();
439        mPaused = true;
440    }
441}
442
443/*
444 * returns true if we should request more data
445 */
446bool NuPlayer::Decoder::doRequestBuffers() {
447    // mRenderer is only NULL if we have a legacy widevine source that
448    // is not yet ready. In this case we must not fetch input.
449    if (isDiscontinuityPending() || mRenderer == NULL) {
450        return false;
451    }
452    status_t err = OK;
453    while (err == OK && !mDequeuedInputBuffers.empty()) {
454        size_t bufferIx = *mDequeuedInputBuffers.begin();
455        sp<AMessage> msg = new AMessage();
456        msg->setSize("buffer-ix", bufferIx);
457        err = fetchInputData(msg);
458        if (err != OK && err != ERROR_END_OF_STREAM) {
459            // if EOS, need to queue EOS buffer
460            break;
461        }
462        mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
463
464        if (!mPendingInputMessages.empty()
465                || !onInputBufferFetched(msg)) {
466            mPendingInputMessages.push_back(msg);
467        }
468    }
469
470    return err == -EWOULDBLOCK
471            && mSource->feedMoreTSData() == OK;
472}
473
474void NuPlayer::Decoder::handleError(int32_t err)
475{
476    // We cannot immediately release the codec due to buffers still outstanding
477    // in the renderer.  We signal to the player the error so it can shutdown/release the
478    // decoder after flushing and increment the generation to discard unnecessary messages.
479
480    ++mBufferGeneration;
481
482    sp<AMessage> notify = mNotify->dup();
483    notify->setInt32("what", kWhatError);
484    notify->setInt32("err", err);
485    notify->post();
486}
487
488bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
489    if (isDiscontinuityPending()) {
490        return false;
491    }
492
493    sp<ABuffer> buffer;
494    mCodec->getInputBuffer(index, &buffer);
495
496    if (buffer == NULL) {
497        handleError(UNKNOWN_ERROR);
498        return false;
499    }
500
501    if (index >= mInputBuffers.size()) {
502        for (size_t i = mInputBuffers.size(); i <= index; ++i) {
503            mInputBuffers.add();
504            mMediaBuffers.add();
505            mInputBufferIsDequeued.add();
506            mMediaBuffers.editItemAt(i) = NULL;
507            mInputBufferIsDequeued.editItemAt(i) = false;
508        }
509    }
510    mInputBuffers.editItemAt(index) = buffer;
511
512    //CHECK_LT(bufferIx, mInputBuffers.size());
513
514    if (mMediaBuffers[index] != NULL) {
515        mMediaBuffers[index]->release();
516        mMediaBuffers.editItemAt(index) = NULL;
517    }
518    mInputBufferIsDequeued.editItemAt(index) = true;
519
520    if (!mCSDsToSubmit.isEmpty()) {
521        sp<AMessage> msg = new AMessage();
522        msg->setSize("buffer-ix", index);
523
524        sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
525        ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
526        msg->setBuffer("buffer", buffer);
527        mCSDsToSubmit.removeAt(0);
528        CHECK(onInputBufferFetched(msg));
529        return true;
530    }
531
532    while (!mPendingInputMessages.empty()) {
533        sp<AMessage> msg = *mPendingInputMessages.begin();
534        if (!onInputBufferFetched(msg)) {
535            break;
536        }
537        mPendingInputMessages.erase(mPendingInputMessages.begin());
538    }
539
540    if (!mInputBufferIsDequeued.editItemAt(index)) {
541        return true;
542    }
543
544    mDequeuedInputBuffers.push_back(index);
545
546    onRequestInputBuffers();
547    return true;
548}
549
550bool NuPlayer::Decoder::handleAnOutputBuffer(
551        size_t index,
552        size_t offset,
553        size_t size,
554        int64_t timeUs,
555        int32_t flags) {
556//    CHECK_LT(bufferIx, mOutputBuffers.size());
557    sp<ABuffer> buffer;
558    mCodec->getOutputBuffer(index, &buffer);
559
560    if (index >= mOutputBuffers.size()) {
561        for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
562            mOutputBuffers.add();
563        }
564    }
565
566    mOutputBuffers.editItemAt(index) = buffer;
567
568    buffer->setRange(offset, size);
569    buffer->meta()->clear();
570    buffer->meta()->setInt64("timeUs", timeUs);
571
572    bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
573    // we do not expect CODECCONFIG or SYNCFRAME for decoder
574
575    sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
576    reply->setSize("buffer-ix", index);
577    reply->setInt32("generation", mBufferGeneration);
578
579    if (eos) {
580        ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
581
582        buffer->meta()->setInt32("eos", true);
583        reply->setInt32("eos", true);
584    } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
585        if (timeUs < mSkipRenderingUntilMediaTimeUs) {
586            ALOGV("[%s] dropping buffer at time %lld as requested.",
587                     mComponentName.c_str(), (long long)timeUs);
588
589            reply->post();
590            return true;
591        }
592
593        mSkipRenderingUntilMediaTimeUs = -1;
594    }
595
596    mNumFramesTotal += !mIsAudio;
597
598    // wait until 1st frame comes out to signal resume complete
599    notifyResumeCompleteIfNecessary();
600
601    if (mRenderer != NULL) {
602        // send the buffer to renderer.
603        mRenderer->queueBuffer(mIsAudio, buffer, reply);
604        if (eos && !isDiscontinuityPending()) {
605            mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
606        }
607    }
608
609    return true;
610}
611
612void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
613    if (!mIsAudio) {
614        int32_t width, height;
615        if (format->findInt32("width", &width)
616                && format->findInt32("height", &height)) {
617            mStats->setInt32("width", width);
618            mStats->setInt32("height", height);
619        }
620        sp<AMessage> notify = mNotify->dup();
621        notify->setInt32("what", kWhatVideoSizeChanged);
622        notify->setMessage("format", format);
623        notify->post();
624    } else if (mRenderer != NULL) {
625        uint32_t flags;
626        int64_t durationUs;
627        bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
628        if (getAudioDeepBufferSetting() // override regardless of source duration
629                || (!hasVideo
630                        && mSource->getDuration(&durationUs) == OK
631                        && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
632            flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
633        } else {
634            flags = AUDIO_OUTPUT_FLAG_NONE;
635        }
636
637        mRenderer->openAudioSink(
638                format, false /* offloadOnly */, hasVideo, flags, NULL /* isOffloaed */);
639    }
640}
641
642void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
643    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
644        if (mMediaBuffers[i] != NULL) {
645            mMediaBuffers[i]->release();
646            mMediaBuffers.editItemAt(i) = NULL;
647        }
648    }
649    mMediaBuffers.resize(mInputBuffers.size());
650    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
651        mMediaBuffers.editItemAt(i) = NULL;
652    }
653    mInputBufferIsDequeued.clear();
654    mInputBufferIsDequeued.resize(mInputBuffers.size());
655    for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
656        mInputBufferIsDequeued.editItemAt(i) = false;
657    }
658
659    mPendingInputMessages.clear();
660    mDequeuedInputBuffers.clear();
661    mSkipRenderingUntilMediaTimeUs = -1;
662}
663
664void NuPlayer::Decoder::requestCodecNotification() {
665    if (mCodec != NULL) {
666        sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
667        reply->setInt32("generation", mBufferGeneration);
668        mCodec->requestActivityNotification(reply);
669    }
670}
671
672bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
673    int32_t generation;
674    CHECK(msg->findInt32("generation", &generation));
675    return generation != mBufferGeneration;
676}
677
678status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
679    sp<ABuffer> accessUnit;
680    bool dropAccessUnit;
681    do {
682        status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
683
684        if (err == -EWOULDBLOCK) {
685            return err;
686        } else if (err != OK) {
687            if (err == INFO_DISCONTINUITY) {
688                int32_t type;
689                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
690
691                bool formatChange =
692                    (mIsAudio &&
693                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
694                    || (!mIsAudio &&
695                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
696
697                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
698
699                ALOGI("%s discontinuity (format=%d, time=%d)",
700                        mIsAudio ? "audio" : "video", formatChange, timeChange);
701
702                bool seamlessFormatChange = false;
703                sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
704                if (formatChange) {
705                    seamlessFormatChange =
706                        supportsSeamlessFormatChange(newFormat);
707                    // treat seamless format change separately
708                    formatChange = !seamlessFormatChange;
709                }
710
711                // For format or time change, return EOS to queue EOS input,
712                // then wait for EOS on output.
713                if (formatChange /* not seamless */) {
714                    mFormatChangePending = true;
715                    err = ERROR_END_OF_STREAM;
716                } else if (timeChange) {
717                    rememberCodecSpecificData(newFormat);
718                    mTimeChangePending = true;
719                    err = ERROR_END_OF_STREAM;
720                } else if (seamlessFormatChange) {
721                    // reuse existing decoder and don't flush
722                    rememberCodecSpecificData(newFormat);
723                    continue;
724                } else {
725                    // This stream is unaffected by the discontinuity
726                    return -EWOULDBLOCK;
727                }
728            }
729
730            // reply should only be returned without a buffer set
731            // when there is an error (including EOS)
732            CHECK(err != OK);
733
734            reply->setInt32("err", err);
735            return ERROR_END_OF_STREAM;
736        }
737
738        dropAccessUnit = false;
739        if (!mIsAudio
740                && !mIsSecure
741                && mRenderer->getVideoLateByUs() > 100000ll
742                && mIsVideoAVC
743                && !IsAVCReferenceFrame(accessUnit)) {
744            dropAccessUnit = true;
745            ++mNumInputFramesDropped;
746        }
747    } while (dropAccessUnit);
748
749    // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
750#if 0
751    int64_t mediaTimeUs;
752    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
753    ALOGV("[%s] feeding input buffer at media time %.3f",
754         mIsAudio ? "audio" : "video",
755         mediaTimeUs / 1E6);
756#endif
757
758    if (mCCDecoder != NULL) {
759        mCCDecoder->decode(accessUnit);
760    }
761
762    reply->setBuffer("buffer", accessUnit);
763
764    return OK;
765}
766
767bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
768    size_t bufferIx;
769    CHECK(msg->findSize("buffer-ix", &bufferIx));
770    CHECK_LT(bufferIx, mInputBuffers.size());
771    sp<ABuffer> codecBuffer = mInputBuffers[bufferIx];
772
773    sp<ABuffer> buffer;
774    bool hasBuffer = msg->findBuffer("buffer", &buffer);
775
776    // handle widevine classic source - that fills an arbitrary input buffer
777    MediaBuffer *mediaBuffer = NULL;
778    if (hasBuffer) {
779        mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase());
780        if (mediaBuffer != NULL) {
781            // likely filled another buffer than we requested: adjust buffer index
782            size_t ix;
783            for (ix = 0; ix < mInputBuffers.size(); ix++) {
784                const sp<ABuffer> &buf = mInputBuffers[ix];
785                if (buf->data() == mediaBuffer->data()) {
786                    // all input buffers are dequeued on start, hence the check
787                    if (!mInputBufferIsDequeued[ix]) {
788                        ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu",
789                                mComponentName.c_str(), ix, bufferIx);
790                        mediaBuffer->release();
791                        return false;
792                    }
793
794                    // TRICKY: need buffer for the metadata, so instead, set
795                    // codecBuffer to the same (though incorrect) buffer to
796                    // avoid a memcpy into the codecBuffer
797                    codecBuffer = buffer;
798                    codecBuffer->setRange(
799                            mediaBuffer->range_offset(),
800                            mediaBuffer->range_length());
801                    bufferIx = ix;
802                    break;
803                }
804            }
805            CHECK(ix < mInputBuffers.size());
806        }
807    }
808
809    if (buffer == NULL /* includes !hasBuffer */) {
810        int32_t streamErr = ERROR_END_OF_STREAM;
811        CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
812
813        CHECK(streamErr != OK);
814
815        // attempt to queue EOS
816        status_t err = mCodec->queueInputBuffer(
817                bufferIx,
818                0,
819                0,
820                0,
821                MediaCodec::BUFFER_FLAG_EOS);
822        if (err == OK) {
823            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
824        } else if (streamErr == ERROR_END_OF_STREAM) {
825            streamErr = err;
826            // err will not be ERROR_END_OF_STREAM
827        }
828
829        if (streamErr != ERROR_END_OF_STREAM) {
830            ALOGE("Stream error for %s (err=%d), EOS %s queued",
831                    mComponentName.c_str(),
832                    streamErr,
833                    err == OK ? "successfully" : "unsuccessfully");
834            handleError(streamErr);
835        }
836    } else {
837        sp<AMessage> extra;
838        if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
839            int64_t resumeAtMediaTimeUs;
840            if (extra->findInt64(
841                        "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
842                ALOGI("[%s] suppressing rendering until %lld us",
843                        mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
844                mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
845            }
846        }
847
848        int64_t timeUs = 0;
849        uint32_t flags = 0;
850        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
851
852        int32_t eos, csd;
853        // we do not expect SYNCFRAME for decoder
854        if (buffer->meta()->findInt32("eos", &eos) && eos) {
855            flags |= MediaCodec::BUFFER_FLAG_EOS;
856        } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
857            flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
858        }
859
860        // copy into codec buffer
861        if (buffer != codecBuffer) {
862            CHECK_LE(buffer->size(), codecBuffer->capacity());
863            codecBuffer->setRange(0, buffer->size());
864            memcpy(codecBuffer->data(), buffer->data(), buffer->size());
865        }
866
867        status_t err = mCodec->queueInputBuffer(
868                        bufferIx,
869                        codecBuffer->offset(),
870                        codecBuffer->size(),
871                        timeUs,
872                        flags);
873        if (err != OK) {
874            if (mediaBuffer != NULL) {
875                mediaBuffer->release();
876            }
877            ALOGE("Failed to queue input buffer for %s (err=%d)",
878                    mComponentName.c_str(), err);
879            handleError(err);
880        } else {
881            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
882            if (mediaBuffer != NULL) {
883                CHECK(mMediaBuffers[bufferIx] == NULL);
884                mMediaBuffers.editItemAt(bufferIx) = mediaBuffer;
885            }
886        }
887    }
888    return true;
889}
890
891void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
892    status_t err;
893    int32_t render;
894    size_t bufferIx;
895    int32_t eos;
896    CHECK(msg->findSize("buffer-ix", &bufferIx));
897
898    if (!mIsAudio) {
899        int64_t timeUs;
900        sp<ABuffer> buffer = mOutputBuffers[bufferIx];
901        buffer->meta()->findInt64("timeUs", &timeUs);
902
903        if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
904            mCCDecoder->display(timeUs);
905        }
906    }
907
908    if (msg->findInt32("render", &render) && render) {
909        int64_t timestampNs;
910        CHECK(msg->findInt64("timestampNs", &timestampNs));
911        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
912    } else {
913        mNumOutputFramesDropped += !mIsAudio;
914        err = mCodec->releaseOutputBuffer(bufferIx);
915    }
916    if (err != OK) {
917        ALOGE("failed to release output buffer for %s (err=%d)",
918                mComponentName.c_str(), err);
919        handleError(err);
920    }
921    if (msg->findInt32("eos", &eos) && eos
922            && isDiscontinuityPending()) {
923        finishHandleDiscontinuity(true /* flushOnTimeChange */);
924    }
925}
926
927bool NuPlayer::Decoder::isDiscontinuityPending() const {
928    return mFormatChangePending || mTimeChangePending;
929}
930
931void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
932    ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
933            mFormatChangePending, mTimeChangePending, flushOnTimeChange);
934
935    // If we have format change, pause and wait to be killed;
936    // If we have time change only, flush and restart fetching.
937
938    if (mFormatChangePending) {
939        mPaused = true;
940    } else if (mTimeChangePending) {
941        if (flushOnTimeChange) {
942            doFlush(false /* notifyComplete */);
943            signalResume(false /* notifyComplete */);
944        }
945    }
946
947    // Notify NuPlayer to either shutdown decoder, or rescan sources
948    sp<AMessage> msg = mNotify->dup();
949    msg->setInt32("what", kWhatInputDiscontinuity);
950    msg->setInt32("formatChange", mFormatChangePending);
951    msg->post();
952
953    mFormatChangePending = false;
954    mTimeChangePending = false;
955}
956
957bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
958        const sp<AMessage> &targetFormat) const {
959    if (targetFormat == NULL) {
960        return true;
961    }
962
963    AString mime;
964    if (!targetFormat->findString("mime", &mime)) {
965        return false;
966    }
967
968    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
969        // field-by-field comparison
970        const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
971        for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
972            int32_t oldVal, newVal;
973            if (!mInputFormat->findInt32(keys[i], &oldVal) ||
974                    !targetFormat->findInt32(keys[i], &newVal) ||
975                    oldVal != newVal) {
976                return false;
977            }
978        }
979
980        sp<ABuffer> oldBuf, newBuf;
981        if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
982                targetFormat->findBuffer("csd-0", &newBuf)) {
983            if (oldBuf->size() != newBuf->size()) {
984                return false;
985            }
986            return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
987        }
988    }
989    return false;
990}
991
992bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
993    if (mInputFormat == NULL) {
994        return false;
995    }
996
997    if (targetFormat == NULL) {
998        return true;
999    }
1000
1001    AString oldMime, newMime;
1002    if (!mInputFormat->findString("mime", &oldMime)
1003            || !targetFormat->findString("mime", &newMime)
1004            || !(oldMime == newMime)) {
1005        return false;
1006    }
1007
1008    bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
1009    bool seamless;
1010    if (audio) {
1011        seamless = supportsSeamlessAudioFormatChange(targetFormat);
1012    } else {
1013        int32_t isAdaptive;
1014        seamless = (mCodec != NULL &&
1015                mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
1016                isAdaptive);
1017    }
1018
1019    ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
1020    return seamless;
1021}
1022
1023void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
1024    if (format == NULL) {
1025        return;
1026    }
1027    mCSDsForCurrentFormat.clear();
1028    for (int32_t i = 0; ; ++i) {
1029        AString tag = "csd-";
1030        tag.append(i);
1031        sp<ABuffer> buffer;
1032        if (!format->findBuffer(tag.c_str(), &buffer)) {
1033            break;
1034        }
1035        mCSDsForCurrentFormat.push(buffer);
1036    }
1037}
1038
1039void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
1040    if (mResumePending) {
1041        mResumePending = false;
1042
1043        sp<AMessage> notify = mNotify->dup();
1044        notify->setInt32("what", kWhatResumeCompleted);
1045        notify->post();
1046    }
1047}
1048
1049}  // namespace android
1050
1051