NuPlayerDecoder.cpp revision c6cfd70f24a11b946859485ce398a189c301a4e2 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerDecoder"
19#include <utils/Log.h>
20#include <inttypes.h>
21
22#include "NuPlayerDecoder.h"
23
24#include "NuPlayerRenderer.h"
25#include "NuPlayerSource.h"
26
27#include <media/ICrypto.h>
28#include <media/stagefright/foundation/ABitReader.h>
29#include <media/stagefright/foundation/ABuffer.h>
30#include <media/stagefright/foundation/ADebug.h>
31#include <media/stagefright/foundation/AMessage.h>
32#include <media/stagefright/MediaBuffer.h>
33#include <media/stagefright/MediaCodec.h>
34#include <media/stagefright/MediaDefs.h>
35#include <media/stagefright/MediaErrors.h>
36
37namespace android {
38
39NuPlayer::Decoder::Decoder(
40        const sp<AMessage> &notify,
41        const sp<Source> &source,
42        const sp<Renderer> &renderer,
43        const sp<NativeWindowWrapper> &nativeWindow)
44    : mNotify(notify),
45      mNativeWindow(nativeWindow),
46      mSource(source),
47      mRenderer(renderer),
48      mSkipRenderingUntilMediaTimeUs(-1ll),
49      mBufferGeneration(0),
50      mPaused(true),
51      mComponentName("decoder") {
52    // Every decoder has its own looper because MediaCodec operations
53    // are blocking, but NuPlayer needs asynchronous operations.
54    mDecoderLooper = new ALooper;
55    mDecoderLooper->setName("NPDecoder");
56    mDecoderLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
57
58    mCodecLooper = new ALooper;
59    mCodecLooper->setName("NPDecoder-CL");
60    mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
61}
62
63NuPlayer::Decoder::~Decoder() {
64    mDecoderLooper->unregisterHandler(id());
65    mDecoderLooper->stop();
66
67    releaseAndResetMediaBuffers();
68}
69
70static
71status_t PostAndAwaitResponse(
72        const sp<AMessage> &msg, sp<AMessage> *response) {
73    status_t err = msg->postAndAwaitResponse(response);
74
75    if (err != OK) {
76        return err;
77    }
78
79    if (!(*response)->findInt32("err", &err)) {
80        err = OK;
81    }
82
83    return err;
84}
85
86void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
87    mCSDsForCurrentFormat.clear();
88    for (int32_t i = 0; ; ++i) {
89        AString tag = "csd-";
90        tag.append(i);
91        sp<ABuffer> buffer;
92        if (!format->findBuffer(tag.c_str(), &buffer)) {
93            break;
94        }
95        mCSDsForCurrentFormat.push(buffer);
96    }
97}
98
99void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
100    CHECK(mCodec == NULL);
101
102    ++mBufferGeneration;
103
104    AString mime;
105    CHECK(format->findString("mime", &mime));
106
107    sp<Surface> surface = NULL;
108    if (mNativeWindow != NULL) {
109        surface = mNativeWindow->getSurfaceTextureClient();
110    }
111
112    mComponentName = mime;
113    mComponentName.append(" decoder");
114    ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), surface.get());
115
116    mCodec = MediaCodec::CreateByType(mCodecLooper, mime.c_str(), false /* encoder */);
117    int32_t secure = 0;
118    if (format->findInt32("secure", &secure) && secure != 0) {
119        if (mCodec != NULL) {
120            mCodec->getName(&mComponentName);
121            mComponentName.append(".secure");
122            mCodec->release();
123            ALOGI("[%s] creating", mComponentName.c_str());
124            mCodec = MediaCodec::CreateByComponentName(
125                    mCodecLooper, mComponentName.c_str());
126        }
127    }
128    if (mCodec == NULL) {
129        ALOGE("Failed to create %s%s decoder",
130                (secure ? "secure " : ""), mime.c_str());
131        handleError(UNKNOWN_ERROR);
132        return;
133    }
134
135    mCodec->getName(&mComponentName);
136
137    status_t err;
138    if (mNativeWindow != NULL) {
139        // disconnect from surface as MediaCodec will reconnect
140        err = native_window_api_disconnect(
141                surface.get(), NATIVE_WINDOW_API_MEDIA);
142        // We treat this as a warning, as this is a preparatory step.
143        // Codec will try to connect to the surface, which is where
144        // any error signaling will occur.
145        ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
146    }
147    err = mCodec->configure(
148            format, surface, NULL /* crypto */, 0 /* flags */);
149    if (err != OK) {
150        ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
151        mCodec->release();
152        mCodec.clear();
153        handleError(err);
154        return;
155    }
156    rememberCodecSpecificData(format);
157
158    // the following should work in configured state
159    CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
160    CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
161
162    err = mCodec->start();
163    if (err != OK) {
164        ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
165        mCodec->release();
166        mCodec.clear();
167        handleError(err);
168        return;
169    }
170
171    // the following should work after start
172    CHECK_EQ((status_t)OK, mCodec->getInputBuffers(&mInputBuffers));
173    releaseAndResetMediaBuffers();
174    CHECK_EQ((status_t)OK, mCodec->getOutputBuffers(&mOutputBuffers));
175    ALOGV("[%s] got %zu input and %zu output buffers",
176            mComponentName.c_str(),
177            mInputBuffers.size(),
178            mOutputBuffers.size());
179
180    if (mRenderer != NULL) {
181        requestCodecNotification();
182    }
183    mPaused = false;
184}
185
186void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
187    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
188        if (mMediaBuffers[i] != NULL) {
189            mMediaBuffers[i]->release();
190            mMediaBuffers.editItemAt(i) = NULL;
191        }
192    }
193    mMediaBuffers.resize(mInputBuffers.size());
194    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
195        mMediaBuffers.editItemAt(i) = NULL;
196    }
197    mInputBufferIsDequeued.clear();
198    mInputBufferIsDequeued.resize(mInputBuffers.size());
199    for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
200        mInputBufferIsDequeued.editItemAt(i) = false;
201    }
202
203    mPendingInputMessages.clear();
204    mSkipRenderingUntilMediaTimeUs = -1;
205}
206
207void NuPlayer::Decoder::requestCodecNotification() {
208    if (mCodec != NULL) {
209        sp<AMessage> reply = new AMessage(kWhatCodecNotify, id());
210        reply->setInt32("generation", mBufferGeneration);
211        mCodec->requestActivityNotification(reply);
212    }
213}
214
215bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
216    int32_t generation;
217    CHECK(msg->findInt32("generation", &generation));
218    return generation != mBufferGeneration;
219}
220
221void NuPlayer::Decoder::init() {
222    mDecoderLooper->registerHandler(this);
223}
224
225void NuPlayer::Decoder::configure(const sp<AMessage> &format) {
226    sp<AMessage> msg = new AMessage(kWhatConfigure, id());
227    msg->setMessage("format", format);
228    msg->post();
229}
230
231void NuPlayer::Decoder::setRenderer(const sp<Renderer> &renderer) {
232    sp<AMessage> msg = new AMessage(kWhatSetRenderer, id());
233    msg->setObject("renderer", renderer);
234    msg->post();
235}
236
237void NuPlayer::Decoder::signalUpdateFormat(const sp<AMessage> &format) {
238    sp<AMessage> msg = new AMessage(kWhatUpdateFormat, id());
239    msg->setMessage("format", format);
240    msg->post();
241}
242
243status_t NuPlayer::Decoder::getInputBuffers(Vector<sp<ABuffer> > *buffers) const {
244    sp<AMessage> msg = new AMessage(kWhatGetInputBuffers, id());
245    msg->setPointer("buffers", buffers);
246
247    sp<AMessage> response;
248    return PostAndAwaitResponse(msg, &response);
249}
250
251void NuPlayer::Decoder::handleError(int32_t err)
252{
253    // We cannot immediately release the codec due to buffers still outstanding
254    // in the renderer.  We signal to the player the error so it can shutdown/release the
255    // decoder after flushing and increment the generation to discard unnecessary messages.
256
257    ++mBufferGeneration;
258
259    sp<AMessage> notify = mNotify->dup();
260    notify->setInt32("what", kWhatError);
261    notify->setInt32("err", err);
262    notify->post();
263}
264
265bool NuPlayer::Decoder::handleAnInputBuffer() {
266    size_t bufferIx = -1;
267    status_t res = mCodec->dequeueInputBuffer(&bufferIx);
268    ALOGV("[%s] dequeued input: %d",
269            mComponentName.c_str(), res == OK ? (int)bufferIx : res);
270    if (res != OK) {
271        if (res != -EAGAIN) {
272            ALOGE("Failed to dequeue input buffer for %s (err=%d)",
273                    mComponentName.c_str(), res);
274            handleError(res);
275        }
276        return false;
277    }
278
279    CHECK_LT(bufferIx, mInputBuffers.size());
280
281    if (mMediaBuffers[bufferIx] != NULL) {
282        mMediaBuffers[bufferIx]->release();
283        mMediaBuffers.editItemAt(bufferIx) = NULL;
284    }
285    mInputBufferIsDequeued.editItemAt(bufferIx) = true;
286
287    sp<AMessage> reply = new AMessage(kWhatInputBufferFilled, id());
288    reply->setSize("buffer-ix", bufferIx);
289    reply->setInt32("generation", mBufferGeneration);
290
291    if (!mCSDsToSubmit.isEmpty()) {
292        sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
293        ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
294        reply->setBuffer("buffer", buffer);
295        mCSDsToSubmit.removeAt(0);
296        CHECK(onInputBufferFilled(reply));
297        return true;
298    }
299
300    while (!mPendingInputMessages.empty()) {
301        sp<AMessage> msg = *mPendingInputMessages.begin();
302        if (!onInputBufferFilled(msg)) {
303            break;
304        }
305        mPendingInputMessages.erase(mPendingInputMessages.begin());
306    }
307
308    if (!mInputBufferIsDequeued.editItemAt(bufferIx)) {
309        return true;
310    }
311
312    sp<AMessage> notify = mNotify->dup();
313    notify->setInt32("what", kWhatFillThisBuffer);
314    notify->setBuffer("buffer", mInputBuffers[bufferIx]);
315    notify->setMessage("reply", reply);
316    notify->post();
317    return true;
318}
319
320bool android::NuPlayer::Decoder::onInputBufferFilled(const sp<AMessage> &msg) {
321    size_t bufferIx;
322    CHECK(msg->findSize("buffer-ix", &bufferIx));
323    CHECK_LT(bufferIx, mInputBuffers.size());
324    sp<ABuffer> codecBuffer = mInputBuffers[bufferIx];
325
326    sp<ABuffer> buffer;
327    bool hasBuffer = msg->findBuffer("buffer", &buffer);
328
329    // handle widevine classic source - that fills an arbitrary input buffer
330    MediaBuffer *mediaBuffer = NULL;
331    if (hasBuffer) {
332        mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase());
333        if (mediaBuffer != NULL) {
334            // likely filled another buffer than we requested: adjust buffer index
335            size_t ix;
336            for (ix = 0; ix < mInputBuffers.size(); ix++) {
337                const sp<ABuffer> &buf = mInputBuffers[ix];
338                if (buf->data() == mediaBuffer->data()) {
339                    // all input buffers are dequeued on start, hence the check
340                    if (!mInputBufferIsDequeued[ix]) {
341                        ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu",
342                                mComponentName.c_str(), ix, bufferIx);
343                        mediaBuffer->release();
344                        return false;
345                    }
346
347                    // TRICKY: need buffer for the metadata, so instead, set
348                    // codecBuffer to the same (though incorrect) buffer to
349                    // avoid a memcpy into the codecBuffer
350                    codecBuffer = buffer;
351                    codecBuffer->setRange(
352                            mediaBuffer->range_offset(),
353                            mediaBuffer->range_length());
354                    bufferIx = ix;
355                    break;
356                }
357            }
358            CHECK(ix < mInputBuffers.size());
359        }
360    }
361
362    if (buffer == NULL /* includes !hasBuffer */) {
363        int32_t streamErr = ERROR_END_OF_STREAM;
364        CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
365
366        if (streamErr == OK) {
367            /* buffers are returned to hold on to */
368            return true;
369        }
370
371        // attempt to queue EOS
372        status_t err = mCodec->queueInputBuffer(
373                bufferIx,
374                0,
375                0,
376                0,
377                MediaCodec::BUFFER_FLAG_EOS);
378        if (err == OK) {
379            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
380        } else if (streamErr == ERROR_END_OF_STREAM) {
381            streamErr = err;
382            // err will not be ERROR_END_OF_STREAM
383        }
384
385        if (streamErr != ERROR_END_OF_STREAM) {
386            ALOGE("Stream error for %s (err=%d), EOS %s queued",
387                    mComponentName.c_str(),
388                    streamErr,
389                    err == OK ? "successfully" : "unsuccessfully");
390            handleError(streamErr);
391        }
392    } else {
393        sp<AMessage> extra;
394        if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
395            int64_t resumeAtMediaTimeUs;
396            if (extra->findInt64(
397                        "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
398                ALOGI("[%s] suppressing rendering until %lld us",
399                        mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
400                mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
401            }
402        }
403
404        int64_t timeUs = 0;
405        uint32_t flags = 0;
406        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
407
408        int32_t eos, csd;
409        // we do not expect SYNCFRAME for decoder
410        if (buffer->meta()->findInt32("eos", &eos) && eos) {
411            flags |= MediaCodec::BUFFER_FLAG_EOS;
412        } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
413            flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
414        }
415
416        // copy into codec buffer
417        if (buffer != codecBuffer) {
418            CHECK_LE(buffer->size(), codecBuffer->capacity());
419            codecBuffer->setRange(0, buffer->size());
420            memcpy(codecBuffer->data(), buffer->data(), buffer->size());
421        }
422
423        status_t err = mCodec->queueInputBuffer(
424                        bufferIx,
425                        codecBuffer->offset(),
426                        codecBuffer->size(),
427                        timeUs,
428                        flags);
429        if (err != OK) {
430            if (mediaBuffer != NULL) {
431                mediaBuffer->release();
432            }
433            ALOGE("Failed to queue input buffer for %s (err=%d)",
434                    mComponentName.c_str(), err);
435            handleError(err);
436        } else {
437            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
438            if (mediaBuffer != NULL) {
439                CHECK(mMediaBuffers[bufferIx] == NULL);
440                mMediaBuffers.editItemAt(bufferIx) = mediaBuffer;
441            }
442        }
443    }
444    return true;
445}
446
447bool NuPlayer::Decoder::handleAnOutputBuffer() {
448    size_t bufferIx = -1;
449    size_t offset;
450    size_t size;
451    int64_t timeUs;
452    uint32_t flags;
453    status_t res = mCodec->dequeueOutputBuffer(
454            &bufferIx, &offset, &size, &timeUs, &flags);
455
456    if (res != OK) {
457        ALOGV("[%s] dequeued output: %d", mComponentName.c_str(), res);
458    } else {
459        ALOGV("[%s] dequeued output: %d (time=%lld flags=%" PRIu32 ")",
460                mComponentName.c_str(), (int)bufferIx, timeUs, flags);
461    }
462
463    if (res == INFO_OUTPUT_BUFFERS_CHANGED) {
464        res = mCodec->getOutputBuffers(&mOutputBuffers);
465        if (res != OK) {
466            ALOGE("Failed to get output buffers for %s after INFO event (err=%d)",
467                    mComponentName.c_str(), res);
468            handleError(res);
469            return false;
470        }
471        // NuPlayer ignores this
472        return true;
473    } else if (res == INFO_FORMAT_CHANGED) {
474        sp<AMessage> format = new AMessage();
475        res = mCodec->getOutputFormat(&format);
476        if (res != OK) {
477            ALOGE("Failed to get output format for %s after INFO event (err=%d)",
478                    mComponentName.c_str(), res);
479            handleError(res);
480            return false;
481        }
482
483        if (isVideo()) {
484            sp<AMessage> notify = mNotify->dup();
485            notify->setInt32("what", kWhatVideoSizeChanged);
486            notify->setMessage("format", format);
487            notify->post();
488        } else if (mRenderer != NULL) {
489            uint32_t flags;
490            int64_t durationUs;
491            bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
492            if (!hasVideo &&
493                    mSource->getDuration(&durationUs) == OK &&
494                    durationUs
495                        > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
496                flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
497            } else {
498                flags = AUDIO_OUTPUT_FLAG_NONE;
499            }
500
501            mRenderer->openAudioSink(
502                    format, false /* offloadOnly */, hasVideo, flags);
503        }
504        return true;
505    } else if (res == INFO_DISCONTINUITY) {
506        // nothing to do
507        return true;
508    } else if (res != OK) {
509        if (res != -EAGAIN) {
510            ALOGE("Failed to dequeue output buffer for %s (err=%d)",
511                    mComponentName.c_str(), res);
512            handleError(res);
513        }
514        return false;
515    }
516
517    CHECK_LT(bufferIx, mOutputBuffers.size());
518    sp<ABuffer> buffer = mOutputBuffers[bufferIx];
519    buffer->setRange(offset, size);
520    buffer->meta()->clear();
521    buffer->meta()->setInt64("timeUs", timeUs);
522    if (flags & MediaCodec::BUFFER_FLAG_EOS) {
523        buffer->meta()->setInt32("eos", true);
524    }
525    // we do not expect CODECCONFIG or SYNCFRAME for decoder
526
527    sp<AMessage> reply = new AMessage(kWhatRenderBuffer, id());
528    reply->setSize("buffer-ix", bufferIx);
529    reply->setInt32("generation", mBufferGeneration);
530
531    if (mSkipRenderingUntilMediaTimeUs >= 0) {
532        if (timeUs < mSkipRenderingUntilMediaTimeUs) {
533            ALOGV("[%s] dropping buffer at time %lld as requested.",
534                     mComponentName.c_str(), (long long)timeUs);
535
536            reply->post();
537            return true;
538        }
539
540        mSkipRenderingUntilMediaTimeUs = -1;
541    }
542
543    if (mRenderer != NULL) {
544        // send the buffer to renderer.
545        mRenderer->queueBuffer(!isVideo(), buffer, reply);
546        if (flags & MediaCodec::BUFFER_FLAG_EOS) {
547            mRenderer->queueEOS(!isVideo(), ERROR_END_OF_STREAM);
548        }
549    }
550
551    return true;
552}
553
554void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
555    status_t err;
556    int32_t render;
557    size_t bufferIx;
558    CHECK(msg->findSize("buffer-ix", &bufferIx));
559
560    if (isVideo()) {
561        int64_t timeUs;
562        sp<ABuffer> buffer = mOutputBuffers[bufferIx];
563        buffer->meta()->findInt64("timeUs", &timeUs);
564        sp<AMessage> notify = mNotify->dup();
565        notify->setInt32("what", kWhatRenderBufferTime);
566        notify->setInt64("timeUs", timeUs);
567        notify->post();
568    }
569
570    if (msg->findInt32("render", &render) && render) {
571        int64_t timestampNs;
572        CHECK(msg->findInt64("timestampNs", &timestampNs));
573        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
574    } else {
575        err = mCodec->releaseOutputBuffer(bufferIx);
576    }
577    if (err != OK) {
578        ALOGE("failed to release output buffer for %s (err=%d)",
579                mComponentName.c_str(), err);
580        handleError(err);
581    }
582}
583
584void NuPlayer::Decoder::onFlush() {
585    if (mRenderer != NULL) {
586        mRenderer->flush(!isVideo());
587    }
588
589    status_t err = OK;
590    if (mCodec != NULL) {
591        err = mCodec->flush();
592        mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
593        ++mBufferGeneration;
594    }
595
596    if (err != OK) {
597        ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
598        handleError(err);
599        // finish with posting kWhatFlushCompleted.
600        // we attempt to release the buffers even if flush fails.
601    }
602    releaseAndResetMediaBuffers();
603
604    sp<AMessage> notify = mNotify->dup();
605    notify->setInt32("what", kWhatFlushCompleted);
606    notify->post();
607    mPaused = true;
608}
609
610void NuPlayer::Decoder::onResume() {
611    mPaused = false;
612}
613
614void NuPlayer::Decoder::onShutdown() {
615    status_t err = OK;
616    if (mCodec != NULL) {
617        err = mCodec->release();
618        mCodec = NULL;
619        ++mBufferGeneration;
620
621        if (mNativeWindow != NULL) {
622            // reconnect to surface as MediaCodec disconnected from it
623            status_t error =
624                    native_window_api_connect(
625                            mNativeWindow->getNativeWindow().get(),
626                            NATIVE_WINDOW_API_MEDIA);
627            ALOGW_IF(error != NO_ERROR,
628                    "[%s] failed to connect to native window, error=%d",
629                    mComponentName.c_str(), error);
630        }
631        mComponentName = "decoder";
632    }
633
634    releaseAndResetMediaBuffers();
635
636    if (err != OK) {
637        ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
638        handleError(err);
639        // finish with posting kWhatShutdownCompleted.
640    }
641
642    sp<AMessage> notify = mNotify->dup();
643    notify->setInt32("what", kWhatShutdownCompleted);
644    notify->post();
645    mPaused = true;
646}
647
648void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
649    ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
650
651    switch (msg->what()) {
652        case kWhatConfigure:
653        {
654            sp<AMessage> format;
655            CHECK(msg->findMessage("format", &format));
656            onConfigure(format);
657            break;
658        }
659
660        case kWhatSetRenderer:
661        {
662            bool hadNoRenderer = (mRenderer == NULL);
663            sp<RefBase> obj;
664            CHECK(msg->findObject("renderer", &obj));
665            mRenderer = static_cast<Renderer *>(obj.get());
666            if (hadNoRenderer && mRenderer != NULL) {
667                requestCodecNotification();
668            }
669            break;
670        }
671
672        case kWhatUpdateFormat:
673        {
674            sp<AMessage> format;
675            CHECK(msg->findMessage("format", &format));
676            rememberCodecSpecificData(format);
677            break;
678        }
679
680        case kWhatGetInputBuffers:
681        {
682            uint32_t replyID;
683            CHECK(msg->senderAwaitsResponse(&replyID));
684
685            Vector<sp<ABuffer> > *dstBuffers;
686            CHECK(msg->findPointer("buffers", (void **)&dstBuffers));
687
688            dstBuffers->clear();
689            for (size_t i = 0; i < mInputBuffers.size(); i++) {
690                dstBuffers->push(mInputBuffers[i]);
691            }
692
693            (new AMessage)->postReply(replyID);
694            break;
695        }
696
697        case kWhatCodecNotify:
698        {
699            if (!isStaleReply(msg)) {
700                int32_t numInput, numOutput;
701
702                if (!msg->findInt32("input-buffers", &numInput)) {
703                    numInput = INT32_MAX;
704                }
705
706                if (!msg->findInt32("output-buffers", &numOutput)) {
707                    numOutput = INT32_MAX;
708                }
709
710                if (!mPaused) {
711                    while (numInput-- > 0 && handleAnInputBuffer()) {}
712                }
713
714                while (numOutput-- > 0 && handleAnOutputBuffer()) {}
715            }
716
717            requestCodecNotification();
718            break;
719        }
720
721        case kWhatInputBufferFilled:
722        {
723            if (!isStaleReply(msg)) {
724                if (!mPendingInputMessages.empty()
725                        || !onInputBufferFilled(msg)) {
726                    mPendingInputMessages.push_back(msg);
727                }
728            }
729
730            break;
731        }
732
733        case kWhatRenderBuffer:
734        {
735            if (!isStaleReply(msg)) {
736                onRenderBuffer(msg);
737            }
738            break;
739        }
740
741        case kWhatFlush:
742        {
743            sp<AMessage> format;
744            if (msg->findMessage("new-format", &format)) {
745                rememberCodecSpecificData(format);
746            }
747            onFlush();
748            break;
749        }
750
751        case kWhatResume:
752        {
753            onResume();
754            break;
755        }
756
757        case kWhatShutdown:
758        {
759            onShutdown();
760            break;
761        }
762
763        default:
764            TRESPASS();
765            break;
766    }
767}
768
769void NuPlayer::Decoder::signalFlush(const sp<AMessage> &format) {
770    sp<AMessage> msg = new AMessage(kWhatFlush, id());
771    if (format != NULL) {
772        msg->setMessage("new-format", format);
773    }
774    msg->post();
775}
776
777void NuPlayer::Decoder::signalResume() {
778    (new AMessage(kWhatResume, id()))->post();
779}
780
781void NuPlayer::Decoder::initiateShutdown() {
782    (new AMessage(kWhatShutdown, id()))->post();
783}
784
785bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(const sp<AMessage> &targetFormat) const {
786    if (targetFormat == NULL) {
787        return true;
788    }
789
790    AString mime;
791    if (!targetFormat->findString("mime", &mime)) {
792        return false;
793    }
794
795    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
796        // field-by-field comparison
797        const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
798        for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
799            int32_t oldVal, newVal;
800            if (!mOutputFormat->findInt32(keys[i], &oldVal) ||
801                    !targetFormat->findInt32(keys[i], &newVal) ||
802                    oldVal != newVal) {
803                return false;
804            }
805        }
806
807        sp<ABuffer> oldBuf, newBuf;
808        if (mOutputFormat->findBuffer("csd-0", &oldBuf) &&
809                targetFormat->findBuffer("csd-0", &newBuf)) {
810            if (oldBuf->size() != newBuf->size()) {
811                return false;
812            }
813            return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
814        }
815    }
816    return false;
817}
818
819bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
820    if (mOutputFormat == NULL) {
821        return false;
822    }
823
824    if (targetFormat == NULL) {
825        return true;
826    }
827
828    AString oldMime, newMime;
829    if (!mOutputFormat->findString("mime", &oldMime)
830            || !targetFormat->findString("mime", &newMime)
831            || !(oldMime == newMime)) {
832        return false;
833    }
834
835    bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
836    bool seamless;
837    if (audio) {
838        seamless = supportsSeamlessAudioFormatChange(targetFormat);
839    } else {
840        int32_t isAdaptive;
841        seamless = (mCodec != NULL &&
842                mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
843                isAdaptive);
844    }
845
846    ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
847    return seamless;
848}
849
850bool NuPlayer::Decoder::isVideo() {
851    return mNativeWindow != NULL;
852}
853
854struct CCData {
855    CCData(uint8_t type, uint8_t data1, uint8_t data2)
856        : mType(type), mData1(data1), mData2(data2) {
857    }
858    bool getChannel(size_t *channel) const {
859        if (mData1 >= 0x10 && mData1 <= 0x1f) {
860            *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0);
861            return true;
862        }
863        return false;
864    }
865
866    uint8_t mType;
867    uint8_t mData1;
868    uint8_t mData2;
869};
870
871static bool isNullPad(CCData *cc) {
872    return cc->mData1 < 0x10 && cc->mData2 < 0x10;
873}
874
875static void dumpBytePair(const sp<ABuffer> &ccBuf) {
876    size_t offset = 0;
877    AString out;
878
879    while (offset < ccBuf->size()) {
880        char tmp[128];
881
882        CCData *cc = (CCData *) (ccBuf->data() + offset);
883
884        if (isNullPad(cc)) {
885            // 1 null pad or XDS metadata, ignore
886            offset += sizeof(CCData);
887            continue;
888        }
889
890        if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) {
891            // 2 basic chars
892            sprintf(tmp, "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2);
893        } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
894                 && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) {
895            // 1 special char
896            sprintf(tmp, "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2);
897        } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A)
898                 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
899            // 1 Spanish/French char
900            sprintf(tmp, "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2);
901        } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B)
902                 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
903            // 1 Portuguese/German/Danish char
904            sprintf(tmp, "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2);
905        } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
906                 && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){
907            // Mid-Row Codes (Table 69)
908            sprintf(tmp, "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2);
909        } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c)
910                  && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f)
911                  ||
912                   ((cc->mData1 == 0x17 || cc->mData1 == 0x1f)
913                  && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){
914            // Misc Control Codes (Table 70)
915            sprintf(tmp, "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2);
916        } else if ((cc->mData1 & 0x70) == 0x10
917                && (cc->mData2 & 0x40) == 0x40
918                && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) {
919            // Preamble Address Codes (Table 71)
920            sprintf(tmp, "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2);
921        } else {
922            sprintf(tmp, "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2);
923        }
924
925        if (out.size() > 0) {
926            out.append(", ");
927        }
928
929        out.append(tmp);
930
931        offset += sizeof(CCData);
932    }
933
934    ALOGI("%s", out.c_str());
935}
936
937NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> &notify)
938    : mNotify(notify),
939      mCurrentChannel(0),
940      mSelectedTrack(-1) {
941      for (size_t i = 0; i < sizeof(mTrackIndices)/sizeof(mTrackIndices[0]); ++i) {
942          mTrackIndices[i] = -1;
943      }
944}
945
946size_t NuPlayer::CCDecoder::getTrackCount() const {
947    return mFoundChannels.size();
948}
949
950sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const {
951    if (!isTrackValid(index)) {
952        return NULL;
953    }
954
955    sp<AMessage> format = new AMessage();
956
957    format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE);
958    format->setString("language", "und");
959    format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608);
960    //CC1, field 0 channel 0
961    bool isDefaultAuto = (mFoundChannels[index] == 0);
962    format->setInt32("auto", isDefaultAuto);
963    format->setInt32("default", isDefaultAuto);
964    format->setInt32("forced", 0);
965
966    return format;
967}
968
969status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) {
970    if (!isTrackValid(index)) {
971        return BAD_VALUE;
972    }
973
974    if (select) {
975        if (mSelectedTrack == (ssize_t)index) {
976            ALOGE("track %zu already selected", index);
977            return BAD_VALUE;
978        }
979        ALOGV("selected track %zu", index);
980        mSelectedTrack = index;
981    } else {
982        if (mSelectedTrack != (ssize_t)index) {
983            ALOGE("track %zu is not selected", index);
984            return BAD_VALUE;
985        }
986        ALOGV("unselected track %zu", index);
987        mSelectedTrack = -1;
988    }
989
990    return OK;
991}
992
993bool NuPlayer::CCDecoder::isSelected() const {
994    return mSelectedTrack >= 0 && mSelectedTrack < (int32_t) getTrackCount();
995}
996
997bool NuPlayer::CCDecoder::isTrackValid(size_t index) const {
998    return index < getTrackCount();
999}
1000
1001int32_t NuPlayer::CCDecoder::getTrackIndex(size_t channel) const {
1002    if (channel < sizeof(mTrackIndices)/sizeof(mTrackIndices[0])) {
1003        return mTrackIndices[channel];
1004    }
1005    return -1;
1006}
1007
1008// returns true if a new CC track is found
1009bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) {
1010    int64_t timeUs;
1011    CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
1012
1013    sp<ABuffer> sei;
1014    if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) {
1015        return false;
1016    }
1017
1018    bool trackAdded = false;
1019
1020    NALBitReader br(sei->data() + 1, sei->size() - 1);
1021    // sei_message()
1022    while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message()
1023        uint32_t payload_type = 0;
1024        size_t payload_size = 0;
1025        uint8_t last_byte;
1026
1027        do {
1028            last_byte = br.getBits(8);
1029            payload_type += last_byte;
1030        } while (last_byte == 0xFF);
1031
1032        do {
1033            last_byte = br.getBits(8);
1034            payload_size += last_byte;
1035        } while (last_byte == 0xFF);
1036
1037        // sei_payload()
1038        if (payload_type == 4) {
1039            // user_data_registered_itu_t_t35()
1040
1041            // ATSC A/72: 6.4.2
1042            uint8_t itu_t_t35_country_code = br.getBits(8);
1043            uint16_t itu_t_t35_provider_code = br.getBits(16);
1044            uint32_t user_identifier = br.getBits(32);
1045            uint8_t user_data_type_code = br.getBits(8);
1046
1047            payload_size -= 1 + 2 + 4 + 1;
1048
1049            if (itu_t_t35_country_code == 0xB5
1050                    && itu_t_t35_provider_code == 0x0031
1051                    && user_identifier == 'GA94'
1052                    && user_data_type_code == 0x3) {
1053                // MPEG_cc_data()
1054                // ATSC A/53 Part 4: 6.2.3.1
1055                br.skipBits(1); //process_em_data_flag
1056                bool process_cc_data_flag = br.getBits(1);
1057                br.skipBits(1); //additional_data_flag
1058                size_t cc_count = br.getBits(5);
1059                br.skipBits(8); // em_data;
1060                payload_size -= 2;
1061
1062                if (process_cc_data_flag) {
1063                    AString out;
1064
1065                    sp<ABuffer> ccBuf = new ABuffer(cc_count * sizeof(CCData));
1066                    ccBuf->setRange(0, 0);
1067
1068                    for (size_t i = 0; i < cc_count; i++) {
1069                        uint8_t marker = br.getBits(5);
1070                        CHECK_EQ(marker, 0x1f);
1071
1072                        bool cc_valid = br.getBits(1);
1073                        uint8_t cc_type = br.getBits(2);
1074                        // remove odd parity bit
1075                        uint8_t cc_data_1 = br.getBits(8) & 0x7f;
1076                        uint8_t cc_data_2 = br.getBits(8) & 0x7f;
1077
1078                        if (cc_valid
1079                                && (cc_type == 0 || cc_type == 1)) {
1080                            CCData cc(cc_type, cc_data_1, cc_data_2);
1081                            if (!isNullPad(&cc)) {
1082                                size_t channel;
1083                                if (cc.getChannel(&channel) && getTrackIndex(channel) < 0) {
1084                                    mTrackIndices[channel] = mFoundChannels.size();
1085                                    mFoundChannels.push_back(channel);
1086                                    trackAdded = true;
1087                                }
1088                                memcpy(ccBuf->data() + ccBuf->size(),
1089                                        (void *)&cc, sizeof(cc));
1090                                ccBuf->setRange(0, ccBuf->size() + sizeof(CCData));
1091                            }
1092                        }
1093                    }
1094                    payload_size -= cc_count * 3;
1095
1096                    mCCMap.add(timeUs, ccBuf);
1097                    break;
1098                }
1099            } else {
1100                ALOGV("Malformed SEI payload type 4");
1101            }
1102        } else {
1103            ALOGV("Unsupported SEI payload type %d", payload_type);
1104        }
1105
1106        // skipping remaining bits of this payload
1107        br.skipBits(payload_size * 8);
1108    }
1109
1110    return trackAdded;
1111}
1112
1113sp<ABuffer> NuPlayer::CCDecoder::filterCCBuf(
1114        const sp<ABuffer> &ccBuf, size_t index) {
1115    sp<ABuffer> filteredCCBuf = new ABuffer(ccBuf->size());
1116    filteredCCBuf->setRange(0, 0);
1117
1118    size_t cc_count = ccBuf->size() / sizeof(CCData);
1119    const CCData* cc_data = (const CCData*)ccBuf->data();
1120    for (size_t i = 0; i < cc_count; ++i) {
1121        size_t channel;
1122        if (cc_data[i].getChannel(&channel)) {
1123            mCurrentChannel = channel;
1124        }
1125        if (mCurrentChannel == mFoundChannels[index]) {
1126            memcpy(filteredCCBuf->data() + filteredCCBuf->size(),
1127                    (void *)&cc_data[i], sizeof(CCData));
1128            filteredCCBuf->setRange(0, filteredCCBuf->size() + sizeof(CCData));
1129        }
1130    }
1131
1132    return filteredCCBuf;
1133}
1134
1135void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) {
1136    if (extractFromSEI(accessUnit)) {
1137        ALOGI("Found CEA-608 track");
1138        sp<AMessage> msg = mNotify->dup();
1139        msg->setInt32("what", kWhatTrackAdded);
1140        msg->post();
1141    }
1142    // TODO: extract CC from other sources
1143}
1144
1145void NuPlayer::CCDecoder::display(int64_t timeUs) {
1146    if (!isTrackValid(mSelectedTrack)) {
1147        ALOGE("Could not find current track(index=%d)", mSelectedTrack);
1148        return;
1149    }
1150
1151    ssize_t index = mCCMap.indexOfKey(timeUs);
1152    if (index < 0) {
1153        ALOGV("cc for timestamp %" PRId64 " not found", timeUs);
1154        return;
1155    }
1156
1157    sp<ABuffer> ccBuf = filterCCBuf(mCCMap.valueAt(index), mSelectedTrack);
1158
1159    if (ccBuf->size() > 0) {
1160#if 0
1161        dumpBytePair(ccBuf);
1162#endif
1163
1164        ccBuf->meta()->setInt32("trackIndex", mSelectedTrack);
1165        ccBuf->meta()->setInt64("timeUs", timeUs);
1166        ccBuf->meta()->setInt64("durationUs", 0ll);
1167
1168        sp<AMessage> msg = mNotify->dup();
1169        msg->setInt32("what", kWhatClosedCaptionData);
1170        msg->setBuffer("buffer", ccBuf);
1171        msg->post();
1172    }
1173
1174    // remove all entries before timeUs
1175    mCCMap.removeItemsAt(0, index + 1);
1176}
1177
1178void NuPlayer::CCDecoder::flush() {
1179    mCCMap.clear();
1180}
1181
1182}  // namespace android
1183
1184