NuPlayerDecoder.cpp revision 8818bf1fbb9ea6b65cd61543ed7db986c3d65d13 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerDecoder"
19#include <utils/Log.h>
20#include <inttypes.h>
21
22#include "NuPlayerDecoder.h"
23
24#include <media/ICrypto.h>
25#include <media/stagefright/foundation/ABitReader.h>
26#include <media/stagefright/foundation/ABuffer.h>
27#include <media/stagefright/foundation/ADebug.h>
28#include <media/stagefright/foundation/AMessage.h>
29#include <media/stagefright/MediaBuffer.h>
30#include <media/stagefright/MediaCodec.h>
31#include <media/stagefright/MediaDefs.h>
32#include <media/stagefright/MediaErrors.h>
33
34namespace android {
35
36NuPlayer::Decoder::Decoder(
37        const sp<AMessage> &notify,
38        const sp<NativeWindowWrapper> &nativeWindow)
39    : mNotify(notify),
40      mNativeWindow(nativeWindow),
41      mBufferGeneration(0),
42      mPaused(true),
43      mComponentName("decoder") {
44    // Every decoder has its own looper because MediaCodec operations
45    // are blocking, but NuPlayer needs asynchronous operations.
46    mDecoderLooper = new ALooper;
47    mDecoderLooper->setName("NPDecoder");
48    mDecoderLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
49
50    mCodecLooper = new ALooper;
51    mCodecLooper->setName("NPDecoder-CL");
52    mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
53}
54
55NuPlayer::Decoder::~Decoder() {
56    mDecoderLooper->unregisterHandler(id());
57    mDecoderLooper->stop();
58
59    releaseAndResetMediaBuffers();
60}
61
62static
63status_t PostAndAwaitResponse(
64        const sp<AMessage> &msg, sp<AMessage> *response) {
65    status_t err = msg->postAndAwaitResponse(response);
66
67    if (err != OK) {
68        return err;
69    }
70
71    if (!(*response)->findInt32("err", &err)) {
72        err = OK;
73    }
74
75    return err;
76}
77
78void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
79    mCSDsForCurrentFormat.clear();
80    for (int32_t i = 0; ; ++i) {
81        AString tag = "csd-";
82        tag.append(i);
83        sp<ABuffer> buffer;
84        if (!format->findBuffer(tag.c_str(), &buffer)) {
85            break;
86        }
87        mCSDsForCurrentFormat.push(buffer);
88    }
89}
90
91void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
92    CHECK(mCodec == NULL);
93
94    ++mBufferGeneration;
95
96    AString mime;
97    CHECK(format->findString("mime", &mime));
98
99    sp<Surface> surface = NULL;
100    if (mNativeWindow != NULL) {
101        surface = mNativeWindow->getSurfaceTextureClient();
102    }
103
104    mComponentName = mime;
105    mComponentName.append(" decoder");
106    ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), surface.get());
107
108    mCodec = MediaCodec::CreateByType(mCodecLooper, mime.c_str(), false /* encoder */);
109    int32_t secure = 0;
110    if (format->findInt32("secure", &secure) && secure != 0) {
111        if (mCodec != NULL) {
112            mCodec->getName(&mComponentName);
113            mComponentName.append(".secure");
114            mCodec->release();
115            ALOGI("[%s] creating", mComponentName.c_str());
116            mCodec = MediaCodec::CreateByComponentName(
117                    mCodecLooper, mComponentName.c_str());
118        }
119    }
120    if (mCodec == NULL) {
121        ALOGE("Failed to create %s%s decoder",
122                (secure ? "secure " : ""), mime.c_str());
123        handleError(UNKNOWN_ERROR);
124        return;
125    }
126
127    mCodec->getName(&mComponentName);
128
129    status_t err;
130    if (mNativeWindow != NULL) {
131        // disconnect from surface as MediaCodec will reconnect
132        err = native_window_api_disconnect(
133                surface.get(), NATIVE_WINDOW_API_MEDIA);
134        // We treat this as a warning, as this is a preparatory step.
135        // Codec will try to connect to the surface, which is where
136        // any error signaling will occur.
137        ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
138    }
139    err = mCodec->configure(
140            format, surface, NULL /* crypto */, 0 /* flags */);
141    if (err != OK) {
142        ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
143        mCodec->release();
144        mCodec.clear();
145        handleError(err);
146        return;
147    }
148    rememberCodecSpecificData(format);
149
150    // the following should work in configured state
151    CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
152    CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
153
154    err = mCodec->start();
155    if (err != OK) {
156        ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
157        mCodec->release();
158        mCodec.clear();
159        handleError(err);
160        return;
161    }
162
163    // the following should work after start
164    CHECK_EQ((status_t)OK, mCodec->getInputBuffers(&mInputBuffers));
165    releaseAndResetMediaBuffers();
166    CHECK_EQ((status_t)OK, mCodec->getOutputBuffers(&mOutputBuffers));
167    ALOGV("[%s] got %zu input and %zu output buffers",
168            mComponentName.c_str(),
169            mInputBuffers.size(),
170            mOutputBuffers.size());
171
172    requestCodecNotification();
173    mPaused = false;
174}
175
176void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
177    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
178        if (mMediaBuffers[i] != NULL) {
179            mMediaBuffers[i]->release();
180            mMediaBuffers.editItemAt(i) = NULL;
181        }
182    }
183    mMediaBuffers.resize(mInputBuffers.size());
184    for (size_t i = 0; i < mMediaBuffers.size(); i++) {
185        mMediaBuffers.editItemAt(i) = NULL;
186    }
187    mInputBufferIsDequeued.clear();
188    mInputBufferIsDequeued.resize(mInputBuffers.size());
189    for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
190        mInputBufferIsDequeued.editItemAt(i) = false;
191    }
192
193    mPendingInputMessages.clear();
194}
195
196void NuPlayer::Decoder::requestCodecNotification() {
197    if (mCodec != NULL) {
198        sp<AMessage> reply = new AMessage(kWhatCodecNotify, id());
199        reply->setInt32("generation", mBufferGeneration);
200        mCodec->requestActivityNotification(reply);
201    }
202}
203
204bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
205    int32_t generation;
206    CHECK(msg->findInt32("generation", &generation));
207    return generation != mBufferGeneration;
208}
209
210void NuPlayer::Decoder::init() {
211    mDecoderLooper->registerHandler(this);
212}
213
214void NuPlayer::Decoder::configure(const sp<AMessage> &format) {
215    sp<AMessage> msg = new AMessage(kWhatConfigure, id());
216    msg->setMessage("format", format);
217    msg->post();
218}
219
220void NuPlayer::Decoder::signalUpdateFormat(const sp<AMessage> &format) {
221    sp<AMessage> msg = new AMessage(kWhatUpdateFormat, id());
222    msg->setMessage("format", format);
223    msg->post();
224}
225
226status_t NuPlayer::Decoder::getInputBuffers(Vector<sp<ABuffer> > *buffers) const {
227    sp<AMessage> msg = new AMessage(kWhatGetInputBuffers, id());
228    msg->setPointer("buffers", buffers);
229
230    sp<AMessage> response;
231    return PostAndAwaitResponse(msg, &response);
232}
233
234void NuPlayer::Decoder::handleError(int32_t err)
235{
236    // We cannot immediately release the codec due to buffers still outstanding
237    // in the renderer.  We signal to the player the error so it can shutdown/release the
238    // decoder after flushing and increment the generation to discard unnecessary messages.
239
240    ++mBufferGeneration;
241
242    sp<AMessage> notify = mNotify->dup();
243    notify->setInt32("what", kWhatError);
244    notify->setInt32("err", err);
245    notify->post();
246}
247
248bool NuPlayer::Decoder::handleAnInputBuffer() {
249    size_t bufferIx = -1;
250    status_t res = mCodec->dequeueInputBuffer(&bufferIx);
251    ALOGV("[%s] dequeued input: %d",
252            mComponentName.c_str(), res == OK ? (int)bufferIx : res);
253    if (res != OK) {
254        if (res != -EAGAIN) {
255            ALOGE("Failed to dequeue input buffer for %s (err=%d)",
256                    mComponentName.c_str(), res);
257            handleError(res);
258        }
259        return false;
260    }
261
262    CHECK_LT(bufferIx, mInputBuffers.size());
263
264    if (mMediaBuffers[bufferIx] != NULL) {
265        mMediaBuffers[bufferIx]->release();
266        mMediaBuffers.editItemAt(bufferIx) = NULL;
267    }
268    mInputBufferIsDequeued.editItemAt(bufferIx) = true;
269
270    sp<AMessage> reply = new AMessage(kWhatInputBufferFilled, id());
271    reply->setSize("buffer-ix", bufferIx);
272    reply->setInt32("generation", mBufferGeneration);
273
274    if (!mCSDsToSubmit.isEmpty()) {
275        sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
276        ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
277        reply->setBuffer("buffer", buffer);
278        mCSDsToSubmit.removeAt(0);
279        if (!onInputBufferFilled(reply)) {
280            handleError(UNKNOWN_ERROR);
281            return false;
282        }
283        return true;
284    }
285
286    while (!mPendingInputMessages.empty()) {
287        sp<AMessage> msg = *mPendingInputMessages.begin();
288        if (!onInputBufferFilled(msg)) {
289            break;
290        }
291        mPendingInputMessages.erase(mPendingInputMessages.begin());
292    }
293
294    if (!mInputBufferIsDequeued.editItemAt(bufferIx)) {
295        return true;
296    }
297
298    sp<AMessage> notify = mNotify->dup();
299    notify->setInt32("what", kWhatFillThisBuffer);
300    notify->setBuffer("buffer", mInputBuffers[bufferIx]);
301    notify->setMessage("reply", reply);
302    notify->post();
303    return true;
304}
305
306bool android::NuPlayer::Decoder::onInputBufferFilled(const sp<AMessage> &msg) {
307    size_t bufferIx;
308    CHECK(msg->findSize("buffer-ix", &bufferIx));
309    CHECK_LT(bufferIx, mInputBuffers.size());
310    sp<ABuffer> codecBuffer = mInputBuffers[bufferIx];
311
312    sp<ABuffer> buffer;
313    bool hasBuffer = msg->findBuffer("buffer", &buffer);
314
315    // handle widevine classic source - that fills an arbitrary input buffer
316    MediaBuffer *mediaBuffer = NULL;
317    if (hasBuffer) {
318        mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase());
319        if (mediaBuffer != NULL) {
320            // likely filled another buffer than we requested: adjust buffer index
321            size_t ix;
322            for (ix = 0; ix < mInputBuffers.size(); ix++) {
323                const sp<ABuffer> &buf = mInputBuffers[ix];
324                if (buf->data() == mediaBuffer->data()) {
325                    // all input buffers are dequeued on start, hence the check
326                    if (!mInputBufferIsDequeued[ix]) {
327                        ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu",
328                                mComponentName.c_str(), ix, bufferIx);
329                        mediaBuffer->release();
330                        return false;
331                    }
332
333                    // TRICKY: need buffer for the metadata, so instead, set
334                    // codecBuffer to the same (though incorrect) buffer to
335                    // avoid a memcpy into the codecBuffer
336                    codecBuffer = buffer;
337                    codecBuffer->setRange(
338                            mediaBuffer->range_offset(),
339                            mediaBuffer->range_length());
340                    bufferIx = ix;
341                    break;
342                }
343            }
344            CHECK(ix < mInputBuffers.size());
345        }
346    }
347
348
349
350    if (buffer == NULL /* includes !hasBuffer */) {
351        int32_t streamErr = ERROR_END_OF_STREAM;
352        CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
353
354        if (streamErr == OK) {
355            /* buffers are returned to hold on to */
356            return true;
357        }
358
359        // attempt to queue EOS
360        status_t err = mCodec->queueInputBuffer(
361                bufferIx,
362                0,
363                0,
364                0,
365                MediaCodec::BUFFER_FLAG_EOS);
366        if (err == OK) {
367            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
368        } else if (streamErr == ERROR_END_OF_STREAM) {
369            streamErr = err;
370            // err will not be ERROR_END_OF_STREAM
371        }
372
373        if (streamErr != ERROR_END_OF_STREAM) {
374            ALOGE("Stream error for %s (err=%d), EOS %s queued",
375                    mComponentName.c_str(),
376                    streamErr,
377                    err == OK ? "successfully" : "unsuccessfully");
378            handleError(streamErr);
379        }
380    } else {
381        int64_t timeUs = 0;
382        uint32_t flags = 0;
383        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
384
385        int32_t eos, csd;
386        // we do not expect SYNCFRAME for decoder
387        if (buffer->meta()->findInt32("eos", &eos) && eos) {
388            flags |= MediaCodec::BUFFER_FLAG_EOS;
389        } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
390            flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
391        }
392
393        // copy into codec buffer
394        if (buffer != codecBuffer) {
395            if (buffer->size() > codecBuffer->capacity()) {
396                handleError(ERROR_BUFFER_TOO_SMALL);
397                return false;
398            }
399            codecBuffer->setRange(0, buffer->size());
400            memcpy(codecBuffer->data(), buffer->data(), buffer->size());
401        }
402
403        status_t err = mCodec->queueInputBuffer(
404                        bufferIx,
405                        codecBuffer->offset(),
406                        codecBuffer->size(),
407                        timeUs,
408                        flags);
409        if (err != OK) {
410            if (mediaBuffer != NULL) {
411                mediaBuffer->release();
412            }
413            ALOGE("Failed to queue input buffer for %s (err=%d)",
414                    mComponentName.c_str(), err);
415            handleError(err);
416        } else {
417            mInputBufferIsDequeued.editItemAt(bufferIx) = false;
418            if (mediaBuffer != NULL) {
419                CHECK(mMediaBuffers[bufferIx] == NULL);
420                mMediaBuffers.editItemAt(bufferIx) = mediaBuffer;
421            }
422        }
423    }
424    return true;
425}
426
427bool NuPlayer::Decoder::handleAnOutputBuffer() {
428    size_t bufferIx = -1;
429    size_t offset;
430    size_t size;
431    int64_t timeUs;
432    uint32_t flags;
433    status_t res = mCodec->dequeueOutputBuffer(
434            &bufferIx, &offset, &size, &timeUs, &flags);
435
436    if (res != OK) {
437        ALOGV("[%s] dequeued output: %d", mComponentName.c_str(), res);
438    } else {
439        ALOGV("[%s] dequeued output: %d (time=%lld flags=%" PRIu32 ")",
440                mComponentName.c_str(), (int)bufferIx, timeUs, flags);
441    }
442
443    if (res == INFO_OUTPUT_BUFFERS_CHANGED) {
444        res = mCodec->getOutputBuffers(&mOutputBuffers);
445        if (res != OK) {
446            ALOGE("Failed to get output buffers for %s after INFO event (err=%d)",
447                    mComponentName.c_str(), res);
448            handleError(res);
449            return false;
450        }
451        // NuPlayer ignores this
452        return true;
453    } else if (res == INFO_FORMAT_CHANGED) {
454        sp<AMessage> format = new AMessage();
455        res = mCodec->getOutputFormat(&format);
456        if (res != OK) {
457            ALOGE("Failed to get output format for %s after INFO event (err=%d)",
458                    mComponentName.c_str(), res);
459            handleError(res);
460            return false;
461        }
462
463        sp<AMessage> notify = mNotify->dup();
464        notify->setInt32("what", kWhatOutputFormatChanged);
465        notify->setMessage("format", format);
466        notify->post();
467        return true;
468    } else if (res == INFO_DISCONTINUITY) {
469        // nothing to do
470        return true;
471    } else if (res != OK) {
472        if (res != -EAGAIN) {
473            ALOGE("Failed to dequeue output buffer for %s (err=%d)",
474                    mComponentName.c_str(), res);
475            handleError(res);
476        }
477        return false;
478    }
479
480    CHECK_LT(bufferIx, mOutputBuffers.size());
481    sp<ABuffer> buffer = mOutputBuffers[bufferIx];
482    buffer->setRange(offset, size);
483    buffer->meta()->clear();
484    buffer->meta()->setInt64("timeUs", timeUs);
485    if (flags & MediaCodec::BUFFER_FLAG_EOS) {
486        buffer->meta()->setInt32("eos", true);
487    }
488    // we do not expect CODECCONFIG or SYNCFRAME for decoder
489
490    sp<AMessage> reply = new AMessage(kWhatRenderBuffer, id());
491    reply->setSize("buffer-ix", bufferIx);
492    reply->setInt32("generation", mBufferGeneration);
493
494    sp<AMessage> notify = mNotify->dup();
495    notify->setInt32("what", kWhatDrainThisBuffer);
496    notify->setBuffer("buffer", buffer);
497    notify->setMessage("reply", reply);
498    notify->post();
499
500    // FIXME: This should be handled after rendering is complete,
501    // but Renderer needs it now
502    if (flags & MediaCodec::BUFFER_FLAG_EOS) {
503        ALOGV("queueing eos [%s]", mComponentName.c_str());
504        sp<AMessage> notify = mNotify->dup();
505        notify->setInt32("what", kWhatEOS);
506        notify->setInt32("err", ERROR_END_OF_STREAM);
507        notify->post();
508    }
509    return true;
510}
511
512void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
513    status_t err;
514    int32_t render;
515    size_t bufferIx;
516    CHECK(msg->findSize("buffer-ix", &bufferIx));
517    if (msg->findInt32("render", &render) && render) {
518        int64_t timestampNs;
519        CHECK(msg->findInt64("timestampNs", &timestampNs));
520        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
521    } else {
522        err = mCodec->releaseOutputBuffer(bufferIx);
523    }
524    if (err != OK) {
525        ALOGE("failed to release output buffer for %s (err=%d)",
526                mComponentName.c_str(), err);
527        handleError(err);
528    }
529}
530
531void NuPlayer::Decoder::onFlush() {
532    status_t err = OK;
533    if (mCodec != NULL) {
534        err = mCodec->flush();
535        mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
536        ++mBufferGeneration;
537    }
538
539    if (err != OK) {
540        ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
541        handleError(err);
542        // finish with posting kWhatFlushCompleted.
543        // we attempt to release the buffers even if flush fails.
544    }
545    releaseAndResetMediaBuffers();
546
547    sp<AMessage> notify = mNotify->dup();
548    notify->setInt32("what", kWhatFlushCompleted);
549    notify->post();
550    mPaused = true;
551}
552
553void NuPlayer::Decoder::onResume() {
554    mPaused = false;
555}
556
557void NuPlayer::Decoder::onShutdown() {
558    status_t err = OK;
559    if (mCodec != NULL) {
560        err = mCodec->release();
561        mCodec = NULL;
562        ++mBufferGeneration;
563
564        if (mNativeWindow != NULL) {
565            // reconnect to surface as MediaCodec disconnected from it
566            status_t error =
567                    native_window_api_connect(
568                            mNativeWindow->getNativeWindow().get(),
569                            NATIVE_WINDOW_API_MEDIA);
570            ALOGW_IF(error != NO_ERROR,
571                    "[%s] failed to connect to native window, error=%d",
572                    mComponentName.c_str(), error);
573        }
574        mComponentName = "decoder";
575    }
576
577    releaseAndResetMediaBuffers();
578
579    if (err != OK) {
580        ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
581        handleError(err);
582        // finish with posting kWhatShutdownCompleted.
583    }
584
585    sp<AMessage> notify = mNotify->dup();
586    notify->setInt32("what", kWhatShutdownCompleted);
587    notify->post();
588    mPaused = true;
589}
590
591void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
592    ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
593
594    switch (msg->what()) {
595        case kWhatConfigure:
596        {
597            sp<AMessage> format;
598            CHECK(msg->findMessage("format", &format));
599            onConfigure(format);
600            break;
601        }
602
603        case kWhatUpdateFormat:
604        {
605            sp<AMessage> format;
606            CHECK(msg->findMessage("format", &format));
607            rememberCodecSpecificData(format);
608            break;
609        }
610
611        case kWhatGetInputBuffers:
612        {
613            uint32_t replyID;
614            CHECK(msg->senderAwaitsResponse(&replyID));
615
616            Vector<sp<ABuffer> > *dstBuffers;
617            CHECK(msg->findPointer("buffers", (void **)&dstBuffers));
618
619            dstBuffers->clear();
620            for (size_t i = 0; i < mInputBuffers.size(); i++) {
621                dstBuffers->push(mInputBuffers[i]);
622            }
623
624            (new AMessage)->postReply(replyID);
625            break;
626        }
627
628        case kWhatCodecNotify:
629        {
630            if (!isStaleReply(msg)) {
631                int32_t numInput, numOutput;
632
633                if (!msg->findInt32("input-buffers", &numInput)) {
634                    numInput = INT32_MAX;
635                }
636
637                if (!msg->findInt32("output-buffers", &numOutput)) {
638                    numOutput = INT32_MAX;
639                }
640
641                if (!mPaused) {
642                    while (numInput-- > 0 && handleAnInputBuffer()) {}
643                }
644
645                while (numOutput-- > 0 && handleAnOutputBuffer()) {}
646            }
647
648            requestCodecNotification();
649            break;
650        }
651
652        case kWhatInputBufferFilled:
653        {
654            if (!isStaleReply(msg)) {
655                if (!mPendingInputMessages.empty()
656                        || !onInputBufferFilled(msg)) {
657                    mPendingInputMessages.push_back(msg);
658                }
659            }
660
661            break;
662        }
663
664        case kWhatRenderBuffer:
665        {
666            if (!isStaleReply(msg)) {
667                onRenderBuffer(msg);
668            }
669            break;
670        }
671
672        case kWhatFlush:
673        {
674            sp<AMessage> format;
675            if (msg->findMessage("new-format", &format)) {
676                rememberCodecSpecificData(format);
677            }
678            onFlush();
679            break;
680        }
681
682        case kWhatResume:
683        {
684            onResume();
685            break;
686        }
687
688        case kWhatShutdown:
689        {
690            onShutdown();
691            break;
692        }
693
694        default:
695            TRESPASS();
696            break;
697    }
698}
699
700void NuPlayer::Decoder::signalFlush(const sp<AMessage> &format) {
701    sp<AMessage> msg = new AMessage(kWhatFlush, id());
702    if (format != NULL) {
703        msg->setMessage("new-format", format);
704    }
705    msg->post();
706}
707
708void NuPlayer::Decoder::signalResume() {
709    (new AMessage(kWhatResume, id()))->post();
710}
711
712void NuPlayer::Decoder::initiateShutdown() {
713    (new AMessage(kWhatShutdown, id()))->post();
714}
715
716bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(const sp<AMessage> &targetFormat) const {
717    if (targetFormat == NULL) {
718        return true;
719    }
720
721    AString mime;
722    if (!targetFormat->findString("mime", &mime)) {
723        return false;
724    }
725
726    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
727        // field-by-field comparison
728        const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
729        for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
730            int32_t oldVal, newVal;
731            if (!mOutputFormat->findInt32(keys[i], &oldVal) ||
732                    !targetFormat->findInt32(keys[i], &newVal) ||
733                    oldVal != newVal) {
734                return false;
735            }
736        }
737
738        sp<ABuffer> oldBuf, newBuf;
739        if (mOutputFormat->findBuffer("csd-0", &oldBuf) &&
740                targetFormat->findBuffer("csd-0", &newBuf)) {
741            if (oldBuf->size() != newBuf->size()) {
742                return false;
743            }
744            return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
745        }
746    }
747    return false;
748}
749
750bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
751    if (mOutputFormat == NULL) {
752        return false;
753    }
754
755    if (targetFormat == NULL) {
756        return true;
757    }
758
759    AString oldMime, newMime;
760    if (!mOutputFormat->findString("mime", &oldMime)
761            || !targetFormat->findString("mime", &newMime)
762            || !(oldMime == newMime)) {
763        return false;
764    }
765
766    bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
767    bool seamless;
768    if (audio) {
769        seamless = supportsSeamlessAudioFormatChange(targetFormat);
770    } else {
771        int32_t isAdaptive;
772        seamless = (mCodec != NULL &&
773                mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
774                isAdaptive);
775    }
776
777    ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
778    return seamless;
779}
780
781struct CCData {
782    CCData(uint8_t type, uint8_t data1, uint8_t data2)
783        : mType(type), mData1(data1), mData2(data2) {
784    }
785    bool getChannel(size_t *channel) const {
786        if (mData1 >= 0x10 && mData1 <= 0x1f) {
787            *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0);
788            return true;
789        }
790        return false;
791    }
792
793    uint8_t mType;
794    uint8_t mData1;
795    uint8_t mData2;
796};
797
798static bool isNullPad(CCData *cc) {
799    return cc->mData1 < 0x10 && cc->mData2 < 0x10;
800}
801
802static void dumpBytePair(const sp<ABuffer> &ccBuf) {
803    size_t offset = 0;
804    AString out;
805
806    while (offset < ccBuf->size()) {
807        char tmp[128];
808
809        CCData *cc = (CCData *) (ccBuf->data() + offset);
810
811        if (isNullPad(cc)) {
812            // 1 null pad or XDS metadata, ignore
813            offset += sizeof(CCData);
814            continue;
815        }
816
817        if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) {
818            // 2 basic chars
819            sprintf(tmp, "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2);
820        } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
821                 && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) {
822            // 1 special char
823            sprintf(tmp, "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2);
824        } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A)
825                 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
826            // 1 Spanish/French char
827            sprintf(tmp, "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2);
828        } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B)
829                 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
830            // 1 Portuguese/German/Danish char
831            sprintf(tmp, "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2);
832        } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
833                 && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){
834            // Mid-Row Codes (Table 69)
835            sprintf(tmp, "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2);
836        } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c)
837                  && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f)
838                  ||
839                   ((cc->mData1 == 0x17 || cc->mData1 == 0x1f)
840                  && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){
841            // Misc Control Codes (Table 70)
842            sprintf(tmp, "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2);
843        } else if ((cc->mData1 & 0x70) == 0x10
844                && (cc->mData2 & 0x40) == 0x40
845                && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) {
846            // Preamble Address Codes (Table 71)
847            sprintf(tmp, "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2);
848        } else {
849            sprintf(tmp, "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2);
850        }
851
852        if (out.size() > 0) {
853            out.append(", ");
854        }
855
856        out.append(tmp);
857
858        offset += sizeof(CCData);
859    }
860
861    ALOGI("%s", out.c_str());
862}
863
864NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> &notify)
865    : mNotify(notify),
866      mCurrentChannel(0),
867      mSelectedTrack(-1) {
868      for (size_t i = 0; i < sizeof(mTrackIndices)/sizeof(mTrackIndices[0]); ++i) {
869          mTrackIndices[i] = -1;
870      }
871}
872
873size_t NuPlayer::CCDecoder::getTrackCount() const {
874    return mFoundChannels.size();
875}
876
877sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const {
878    if (!isTrackValid(index)) {
879        return NULL;
880    }
881
882    sp<AMessage> format = new AMessage();
883
884    format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE);
885    format->setString("language", "und");
886    format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608);
887    //CC1, field 0 channel 0
888    bool isDefaultAuto = (mFoundChannels[index] == 0);
889    format->setInt32("auto", isDefaultAuto);
890    format->setInt32("default", isDefaultAuto);
891    format->setInt32("forced", 0);
892
893    return format;
894}
895
896status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) {
897    if (!isTrackValid(index)) {
898        return BAD_VALUE;
899    }
900
901    if (select) {
902        if (mSelectedTrack == (ssize_t)index) {
903            ALOGE("track %zu already selected", index);
904            return BAD_VALUE;
905        }
906        ALOGV("selected track %zu", index);
907        mSelectedTrack = index;
908    } else {
909        if (mSelectedTrack != (ssize_t)index) {
910            ALOGE("track %zu is not selected", index);
911            return BAD_VALUE;
912        }
913        ALOGV("unselected track %zu", index);
914        mSelectedTrack = -1;
915    }
916
917    return OK;
918}
919
920bool NuPlayer::CCDecoder::isSelected() const {
921    return mSelectedTrack >= 0 && mSelectedTrack < (int32_t) getTrackCount();
922}
923
924bool NuPlayer::CCDecoder::isTrackValid(size_t index) const {
925    return index < getTrackCount();
926}
927
928int32_t NuPlayer::CCDecoder::getTrackIndex(size_t channel) const {
929    if (channel < sizeof(mTrackIndices)/sizeof(mTrackIndices[0])) {
930        return mTrackIndices[channel];
931    }
932    return -1;
933}
934
935// returns true if a new CC track is found
936bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) {
937    int64_t timeUs;
938    CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
939
940    sp<ABuffer> sei;
941    if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) {
942        return false;
943    }
944
945    bool trackAdded = false;
946
947    NALBitReader br(sei->data() + 1, sei->size() - 1);
948    // sei_message()
949    while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message()
950        uint32_t payload_type = 0;
951        size_t payload_size = 0;
952        uint8_t last_byte;
953
954        do {
955            last_byte = br.getBits(8);
956            payload_type += last_byte;
957        } while (last_byte == 0xFF);
958
959        do {
960            last_byte = br.getBits(8);
961            payload_size += last_byte;
962        } while (last_byte == 0xFF);
963
964        // sei_payload()
965        if (payload_type == 4) {
966            // user_data_registered_itu_t_t35()
967
968            // ATSC A/72: 6.4.2
969            uint8_t itu_t_t35_country_code = br.getBits(8);
970            uint16_t itu_t_t35_provider_code = br.getBits(16);
971            uint32_t user_identifier = br.getBits(32);
972            uint8_t user_data_type_code = br.getBits(8);
973
974            payload_size -= 1 + 2 + 4 + 1;
975
976            if (itu_t_t35_country_code == 0xB5
977                    && itu_t_t35_provider_code == 0x0031
978                    && user_identifier == 'GA94'
979                    && user_data_type_code == 0x3) {
980                // MPEG_cc_data()
981                // ATSC A/53 Part 4: 6.2.3.1
982                br.skipBits(1); //process_em_data_flag
983                bool process_cc_data_flag = br.getBits(1);
984                br.skipBits(1); //additional_data_flag
985                size_t cc_count = br.getBits(5);
986                br.skipBits(8); // em_data;
987                payload_size -= 2;
988
989                if (process_cc_data_flag) {
990                    AString out;
991
992                    sp<ABuffer> ccBuf = new ABuffer(cc_count * sizeof(CCData));
993                    ccBuf->setRange(0, 0);
994
995                    for (size_t i = 0; i < cc_count; i++) {
996                        uint8_t marker = br.getBits(5);
997                        CHECK_EQ(marker, 0x1f);
998
999                        bool cc_valid = br.getBits(1);
1000                        uint8_t cc_type = br.getBits(2);
1001                        // remove odd parity bit
1002                        uint8_t cc_data_1 = br.getBits(8) & 0x7f;
1003                        uint8_t cc_data_2 = br.getBits(8) & 0x7f;
1004
1005                        if (cc_valid
1006                                && (cc_type == 0 || cc_type == 1)) {
1007                            CCData cc(cc_type, cc_data_1, cc_data_2);
1008                            if (!isNullPad(&cc)) {
1009                                size_t channel;
1010                                if (cc.getChannel(&channel) && getTrackIndex(channel) < 0) {
1011                                    mTrackIndices[channel] = mFoundChannels.size();
1012                                    mFoundChannels.push_back(channel);
1013                                    trackAdded = true;
1014                                }
1015                                memcpy(ccBuf->data() + ccBuf->size(),
1016                                        (void *)&cc, sizeof(cc));
1017                                ccBuf->setRange(0, ccBuf->size() + sizeof(CCData));
1018                            }
1019                        }
1020                    }
1021                    payload_size -= cc_count * 3;
1022
1023                    mCCMap.add(timeUs, ccBuf);
1024                    break;
1025                }
1026            } else {
1027                ALOGV("Malformed SEI payload type 4");
1028            }
1029        } else {
1030            ALOGV("Unsupported SEI payload type %d", payload_type);
1031        }
1032
1033        // skipping remaining bits of this payload
1034        br.skipBits(payload_size * 8);
1035    }
1036
1037    return trackAdded;
1038}
1039
1040sp<ABuffer> NuPlayer::CCDecoder::filterCCBuf(
1041        const sp<ABuffer> &ccBuf, size_t index) {
1042    sp<ABuffer> filteredCCBuf = new ABuffer(ccBuf->size());
1043    filteredCCBuf->setRange(0, 0);
1044
1045    size_t cc_count = ccBuf->size() / sizeof(CCData);
1046    const CCData* cc_data = (const CCData*)ccBuf->data();
1047    for (size_t i = 0; i < cc_count; ++i) {
1048        size_t channel;
1049        if (cc_data[i].getChannel(&channel)) {
1050            mCurrentChannel = channel;
1051        }
1052        if (mCurrentChannel == mFoundChannels[index]) {
1053            memcpy(filteredCCBuf->data() + filteredCCBuf->size(),
1054                    (void *)&cc_data[i], sizeof(CCData));
1055            filteredCCBuf->setRange(0, filteredCCBuf->size() + sizeof(CCData));
1056        }
1057    }
1058
1059    return filteredCCBuf;
1060}
1061
1062void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) {
1063    if (extractFromSEI(accessUnit)) {
1064        ALOGI("Found CEA-608 track");
1065        sp<AMessage> msg = mNotify->dup();
1066        msg->setInt32("what", kWhatTrackAdded);
1067        msg->post();
1068    }
1069    // TODO: extract CC from other sources
1070}
1071
1072void NuPlayer::CCDecoder::display(int64_t timeUs) {
1073    if (!isTrackValid(mSelectedTrack)) {
1074        ALOGE("Could not find current track(index=%d)", mSelectedTrack);
1075        return;
1076    }
1077
1078    ssize_t index = mCCMap.indexOfKey(timeUs);
1079    if (index < 0) {
1080        ALOGV("cc for timestamp %" PRId64 " not found", timeUs);
1081        return;
1082    }
1083
1084    sp<ABuffer> ccBuf = filterCCBuf(mCCMap.valueAt(index), mSelectedTrack);
1085
1086    if (ccBuf->size() > 0) {
1087#if 0
1088        dumpBytePair(ccBuf);
1089#endif
1090
1091        ccBuf->meta()->setInt32("trackIndex", mSelectedTrack);
1092        ccBuf->meta()->setInt64("timeUs", timeUs);
1093        ccBuf->meta()->setInt64("durationUs", 0ll);
1094
1095        sp<AMessage> msg = mNotify->dup();
1096        msg->setInt32("what", kWhatClosedCaptionData);
1097        msg->setBuffer("buffer", ccBuf);
1098        msg->post();
1099    }
1100
1101    // remove all entries before timeUs
1102    mCCMap.removeItemsAt(0, index + 1);
1103}
1104
1105void NuPlayer::CCDecoder::flush() {
1106    mCCMap.clear();
1107}
1108
1109}  // namespace android
1110
1111