MediaCodec.cpp revision 749dafad09d85f2aaf6902a7ff16b4087e3bc4c7 
1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "MediaCodec"
19#include <utils/Log.h>
20#include <inttypes.h>
21
22#include <media/stagefright/MediaCodec.h>
23
24#include "include/SoftwareRenderer.h"
25
26#include <gui/Surface.h>
27#include <media/ICrypto.h>
28#include <media/stagefright/foundation/ABuffer.h>
29#include <media/stagefright/foundation/ADebug.h>
30#include <media/stagefright/foundation/AMessage.h>
31#include <media/stagefright/foundation/AString.h>
32#include <media/stagefright/foundation/hexdump.h>
33#include <media/stagefright/ACodec.h>
34#include <media/stagefright/BufferProducerWrapper.h>
35#include <media/stagefright/MediaCodecList.h>
36#include <media/stagefright/MediaDefs.h>
37#include <media/stagefright/MediaErrors.h>
38#include <media/stagefright/MetaData.h>
39#include <media/stagefright/NativeWindowWrapper.h>
40
41#include "include/avc_utils.h"
42
43namespace android {
44
45// static
46sp<MediaCodec> MediaCodec::CreateByType(
47        const sp<ALooper> &looper, const char *mime, bool encoder) {
48    sp<MediaCodec> codec = new MediaCodec(looper);
49    if (codec->init(mime, true /* nameIsType */, encoder) != OK) {
50        return NULL;
51    }
52
53    return codec;
54}
55
56// static
57sp<MediaCodec> MediaCodec::CreateByComponentName(
58        const sp<ALooper> &looper, const char *name) {
59    sp<MediaCodec> codec = new MediaCodec(looper);
60    if (codec->init(name, false /* nameIsType */, false /* encoder */) != OK) {
61        return NULL;
62    }
63
64    return codec;
65}
66
67MediaCodec::MediaCodec(const sp<ALooper> &looper)
68    : mState(UNINITIALIZED),
69      mLooper(looper),
70      mCodec(NULL),
71      mReplyID(0),
72      mFlags(0),
73      mSoftRenderer(NULL),
74      mDequeueInputTimeoutGeneration(0),
75      mDequeueInputReplyID(0),
76      mDequeueOutputTimeoutGeneration(0),
77      mDequeueOutputReplyID(0),
78      mHaveInputSurface(false) {
79}
80
81MediaCodec::~MediaCodec() {
82    CHECK_EQ(mState, UNINITIALIZED);
83}
84
85// static
86status_t MediaCodec::PostAndAwaitResponse(
87        const sp<AMessage> &msg, sp<AMessage> *response) {
88    status_t err = msg->postAndAwaitResponse(response);
89
90    if (err != OK) {
91        return err;
92    }
93
94    if (!(*response)->findInt32("err", &err)) {
95        err = OK;
96    }
97
98    return err;
99}
100
101// static
102void MediaCodec::PostReplyWithError(int32_t replyID, int32_t err) {
103    sp<AMessage> response = new AMessage;
104    response->setInt32("err", err);
105    response->postReply(replyID);
106}
107
108status_t MediaCodec::init(const char *name, bool nameIsType, bool encoder) {
109    // Current video decoders do not return from OMX_FillThisBuffer
110    // quickly, violating the OpenMAX specs, until that is remedied
111    // we need to invest in an extra looper to free the main event
112    // queue.
113    mCodec = new ACodec;
114    bool needDedicatedLooper = false;
115    if (nameIsType && !strncasecmp(name, "video/", 6)) {
116        needDedicatedLooper = true;
117    } else {
118        AString tmp = name;
119        if (tmp.endsWith(".secure")) {
120            tmp.erase(tmp.size() - 7, 7);
121        }
122        const MediaCodecList *mcl = MediaCodecList::getInstance();
123        ssize_t codecIdx = mcl->findCodecByName(tmp.c_str());
124        if (codecIdx >= 0) {
125            Vector<AString> types;
126            if (mcl->getSupportedTypes(codecIdx, &types) == OK) {
127                for (size_t i = 0; i < types.size(); i++) {
128                    if (types[i].startsWith("video/")) {
129                        needDedicatedLooper = true;
130                        break;
131                    }
132                }
133            }
134        }
135    }
136
137    if (needDedicatedLooper) {
138        if (mCodecLooper == NULL) {
139            mCodecLooper = new ALooper;
140            mCodecLooper->setName("CodecLooper");
141            mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
142        }
143
144        mCodecLooper->registerHandler(mCodec);
145    } else {
146        mLooper->registerHandler(mCodec);
147    }
148
149    mLooper->registerHandler(this);
150
151    mCodec->setNotificationMessage(new AMessage(kWhatCodecNotify, id()));
152
153    sp<AMessage> msg = new AMessage(kWhatInit, id());
154    msg->setString("name", name);
155    msg->setInt32("nameIsType", nameIsType);
156
157    if (nameIsType) {
158        msg->setInt32("encoder", encoder);
159    }
160
161    sp<AMessage> response;
162    return PostAndAwaitResponse(msg, &response);
163}
164
165status_t MediaCodec::setCallback(const sp<AMessage> &callback) {
166    sp<AMessage> msg = new AMessage(kWhatSetCallback, id());
167    msg->setMessage("callback", callback);
168
169    sp<AMessage> response;
170    return PostAndAwaitResponse(msg, &response);
171}
172
173status_t MediaCodec::configure(
174        const sp<AMessage> &format,
175        const sp<Surface> &nativeWindow,
176        const sp<ICrypto> &crypto,
177        uint32_t flags) {
178    sp<AMessage> msg = new AMessage(kWhatConfigure, id());
179
180    msg->setMessage("format", format);
181    msg->setInt32("flags", flags);
182
183    if (nativeWindow != NULL) {
184        msg->setObject(
185                "native-window",
186                new NativeWindowWrapper(nativeWindow));
187    }
188
189    if (crypto != NULL) {
190        msg->setPointer("crypto", crypto.get());
191    }
192
193    sp<AMessage> response;
194    return PostAndAwaitResponse(msg, &response);
195}
196
197status_t MediaCodec::createInputSurface(
198        sp<IGraphicBufferProducer>* bufferProducer) {
199    sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, id());
200
201    sp<AMessage> response;
202    status_t err = PostAndAwaitResponse(msg, &response);
203    if (err == NO_ERROR) {
204        // unwrap the sp<IGraphicBufferProducer>
205        sp<RefBase> obj;
206        bool found = response->findObject("input-surface", &obj);
207        CHECK(found);
208        sp<BufferProducerWrapper> wrapper(
209                static_cast<BufferProducerWrapper*>(obj.get()));
210        *bufferProducer = wrapper->getBufferProducer();
211    } else {
212        ALOGW("createInputSurface failed, err=%d", err);
213    }
214    return err;
215}
216
217status_t MediaCodec::start() {
218    sp<AMessage> msg = new AMessage(kWhatStart, id());
219
220    sp<AMessage> response;
221    return PostAndAwaitResponse(msg, &response);
222}
223
224status_t MediaCodec::stop() {
225    sp<AMessage> msg = new AMessage(kWhatStop, id());
226
227    sp<AMessage> response;
228    return PostAndAwaitResponse(msg, &response);
229}
230
231status_t MediaCodec::release() {
232    sp<AMessage> msg = new AMessage(kWhatRelease, id());
233
234    sp<AMessage> response;
235    return PostAndAwaitResponse(msg, &response);
236}
237
238status_t MediaCodec::queueInputBuffer(
239        size_t index,
240        size_t offset,
241        size_t size,
242        int64_t presentationTimeUs,
243        uint32_t flags,
244        AString *errorDetailMsg) {
245    if (errorDetailMsg != NULL) {
246        errorDetailMsg->clear();
247    }
248
249    sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, id());
250    msg->setSize("index", index);
251    msg->setSize("offset", offset);
252    msg->setSize("size", size);
253    msg->setInt64("timeUs", presentationTimeUs);
254    msg->setInt32("flags", flags);
255    msg->setPointer("errorDetailMsg", errorDetailMsg);
256
257    sp<AMessage> response;
258    return PostAndAwaitResponse(msg, &response);
259}
260
261status_t MediaCodec::queueSecureInputBuffer(
262        size_t index,
263        size_t offset,
264        const CryptoPlugin::SubSample *subSamples,
265        size_t numSubSamples,
266        const uint8_t key[16],
267        const uint8_t iv[16],
268        CryptoPlugin::Mode mode,
269        int64_t presentationTimeUs,
270        uint32_t flags,
271        AString *errorDetailMsg) {
272    if (errorDetailMsg != NULL) {
273        errorDetailMsg->clear();
274    }
275
276    sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, id());
277    msg->setSize("index", index);
278    msg->setSize("offset", offset);
279    msg->setPointer("subSamples", (void *)subSamples);
280    msg->setSize("numSubSamples", numSubSamples);
281    msg->setPointer("key", (void *)key);
282    msg->setPointer("iv", (void *)iv);
283    msg->setInt32("mode", mode);
284    msg->setInt64("timeUs", presentationTimeUs);
285    msg->setInt32("flags", flags);
286    msg->setPointer("errorDetailMsg", errorDetailMsg);
287
288    sp<AMessage> response;
289    status_t err = PostAndAwaitResponse(msg, &response);
290
291    return err;
292}
293
294status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) {
295    sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, id());
296    msg->setInt64("timeoutUs", timeoutUs);
297
298    sp<AMessage> response;
299    status_t err;
300    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
301        return err;
302    }
303
304    CHECK(response->findSize("index", index));
305
306    return OK;
307}
308
309status_t MediaCodec::dequeueOutputBuffer(
310        size_t *index,
311        size_t *offset,
312        size_t *size,
313        int64_t *presentationTimeUs,
314        uint32_t *flags,
315        int64_t timeoutUs) {
316    sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, id());
317    msg->setInt64("timeoutUs", timeoutUs);
318
319    sp<AMessage> response;
320    status_t err;
321    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
322        return err;
323    }
324
325    CHECK(response->findSize("index", index));
326    CHECK(response->findSize("offset", offset));
327    CHECK(response->findSize("size", size));
328    CHECK(response->findInt64("timeUs", presentationTimeUs));
329    CHECK(response->findInt32("flags", (int32_t *)flags));
330
331    return OK;
332}
333
334status_t MediaCodec::renderOutputBufferAndRelease(size_t index) {
335    sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, id());
336    msg->setSize("index", index);
337    msg->setInt32("render", true);
338
339    sp<AMessage> response;
340    return PostAndAwaitResponse(msg, &response);
341}
342
343status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) {
344    sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, id());
345    msg->setSize("index", index);
346    msg->setInt32("render", true);
347    msg->setInt64("timestampNs", timestampNs);
348
349    sp<AMessage> response;
350    return PostAndAwaitResponse(msg, &response);
351}
352
353status_t MediaCodec::releaseOutputBuffer(size_t index) {
354    sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, id());
355    msg->setSize("index", index);
356
357    sp<AMessage> response;
358    return PostAndAwaitResponse(msg, &response);
359}
360
361status_t MediaCodec::signalEndOfInputStream() {
362    sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, id());
363
364    sp<AMessage> response;
365    return PostAndAwaitResponse(msg, &response);
366}
367
368status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const {
369    sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, id());
370
371    sp<AMessage> response;
372    status_t err;
373    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
374        return err;
375    }
376
377    CHECK(response->findMessage("format", format));
378
379    return OK;
380}
381
382status_t MediaCodec::getInputFormat(sp<AMessage> *format) const {
383    sp<AMessage> msg = new AMessage(kWhatGetInputFormat, id());
384
385    sp<AMessage> response;
386    status_t err;
387    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
388        return err;
389    }
390
391    CHECK(response->findMessage("format", format));
392
393    return OK;
394}
395
396status_t MediaCodec::getName(AString *name) const {
397    sp<AMessage> msg = new AMessage(kWhatGetName, id());
398
399    sp<AMessage> response;
400    status_t err;
401    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
402        return err;
403    }
404
405    CHECK(response->findString("name", name));
406
407    return OK;
408}
409
410status_t MediaCodec::getInputBuffers(Vector<sp<ABuffer> > *buffers) const {
411    sp<AMessage> msg = new AMessage(kWhatGetBuffers, id());
412    msg->setInt32("portIndex", kPortIndexInput);
413    msg->setPointer("buffers", buffers);
414
415    sp<AMessage> response;
416    return PostAndAwaitResponse(msg, &response);
417}
418
419status_t MediaCodec::getOutputBuffers(Vector<sp<ABuffer> > *buffers) const {
420    sp<AMessage> msg = new AMessage(kWhatGetBuffers, id());
421    msg->setInt32("portIndex", kPortIndexOutput);
422    msg->setPointer("buffers", buffers);
423
424    sp<AMessage> response;
425    return PostAndAwaitResponse(msg, &response);
426}
427
428status_t MediaCodec::getOutputBuffer(size_t index, sp<ABuffer> *buffer) {
429    sp<AMessage> format;
430    return getBufferAndFormat(kPortIndexOutput, index, buffer, &format);
431}
432
433status_t MediaCodec::getOutputFormat(size_t index, sp<AMessage> *format) {
434    sp<ABuffer> buffer;
435    return getBufferAndFormat(kPortIndexOutput, index, &buffer, format);
436}
437
438status_t MediaCodec::getInputBuffer(size_t index, sp<ABuffer> *buffer) {
439    sp<AMessage> format;
440    return getBufferAndFormat(kPortIndexInput, index, buffer, &format);
441}
442
443status_t MediaCodec::getBufferAndFormat(
444        size_t portIndex, size_t index,
445        sp<ABuffer> *buffer, sp<AMessage> *format) {
446    // use mutex instead of a context switch
447
448    buffer->clear();
449    format->clear();
450    if (mState != STARTED) {
451        return INVALID_OPERATION;
452    }
453
454    // we do not want mPortBuffers to change during this section
455    // we also don't want mOwnedByClient to change during this
456    Mutex::Autolock al(mBufferLock);
457    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
458    if (index < buffers->size()) {
459        const BufferInfo &info = buffers->itemAt(index);
460        if (info.mOwnedByClient) {
461            *buffer = info.mData;
462            *format = info.mFormat;
463        }
464    }
465    return OK;
466}
467
468status_t MediaCodec::flush() {
469    sp<AMessage> msg = new AMessage(kWhatFlush, id());
470
471    sp<AMessage> response;
472    return PostAndAwaitResponse(msg, &response);
473}
474
475status_t MediaCodec::requestIDRFrame() {
476    (new AMessage(kWhatRequestIDRFrame, id()))->post();
477
478    return OK;
479}
480
481void MediaCodec::requestActivityNotification(const sp<AMessage> &notify) {
482    sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, id());
483    msg->setMessage("notify", notify);
484    msg->post();
485}
486
487////////////////////////////////////////////////////////////////////////////////
488
489void MediaCodec::cancelPendingDequeueOperations() {
490    if (mFlags & kFlagDequeueInputPending) {
491        PostReplyWithError(mDequeueInputReplyID, INVALID_OPERATION);
492
493        ++mDequeueInputTimeoutGeneration;
494        mDequeueInputReplyID = 0;
495        mFlags &= ~kFlagDequeueInputPending;
496    }
497
498    if (mFlags & kFlagDequeueOutputPending) {
499        PostReplyWithError(mDequeueOutputReplyID, INVALID_OPERATION);
500
501        ++mDequeueOutputTimeoutGeneration;
502        mDequeueOutputReplyID = 0;
503        mFlags &= ~kFlagDequeueOutputPending;
504    }
505}
506
507bool MediaCodec::handleDequeueInputBuffer(uint32_t replyID, bool newRequest) {
508    if (mState != STARTED
509            || (mFlags & kFlagStickyError)
510            || (newRequest && (mFlags & kFlagDequeueInputPending))) {
511        PostReplyWithError(replyID, INVALID_OPERATION);
512        return true;
513    }
514
515    ssize_t index = dequeuePortBuffer(kPortIndexInput);
516
517    if (index < 0) {
518        CHECK_EQ(index, -EAGAIN);
519        return false;
520    }
521
522    sp<AMessage> response = new AMessage;
523    response->setSize("index", index);
524    response->postReply(replyID);
525
526    return true;
527}
528
529bool MediaCodec::handleDequeueOutputBuffer(uint32_t replyID, bool newRequest) {
530    sp<AMessage> response = new AMessage;
531
532    if (mState != STARTED
533            || (mFlags & kFlagStickyError)
534            || (newRequest && (mFlags & kFlagDequeueOutputPending))) {
535        response->setInt32("err", INVALID_OPERATION);
536    } else if (mFlags & kFlagOutputBuffersChanged) {
537        response->setInt32("err", INFO_OUTPUT_BUFFERS_CHANGED);
538        mFlags &= ~kFlagOutputBuffersChanged;
539    } else if (mFlags & kFlagOutputFormatChanged) {
540        response->setInt32("err", INFO_FORMAT_CHANGED);
541        mFlags &= ~kFlagOutputFormatChanged;
542    } else {
543        ssize_t index = dequeuePortBuffer(kPortIndexOutput);
544
545        if (index < 0) {
546            CHECK_EQ(index, -EAGAIN);
547            return false;
548        }
549
550        const sp<ABuffer> &buffer =
551            mPortBuffers[kPortIndexOutput].itemAt(index).mData;
552
553        response->setSize("index", index);
554        response->setSize("offset", buffer->offset());
555        response->setSize("size", buffer->size());
556
557        int64_t timeUs;
558        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
559
560        response->setInt64("timeUs", timeUs);
561
562        int32_t omxFlags;
563        CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags));
564
565        uint32_t flags = 0;
566        if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) {
567            flags |= BUFFER_FLAG_SYNCFRAME;
568        }
569        if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) {
570            flags |= BUFFER_FLAG_CODECCONFIG;
571        }
572        if (omxFlags & OMX_BUFFERFLAG_EOS) {
573            flags |= BUFFER_FLAG_EOS;
574        }
575
576        response->setInt32("flags", flags);
577    }
578
579    response->postReply(replyID);
580
581    return true;
582}
583
584void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
585    switch (msg->what()) {
586        case kWhatCodecNotify:
587        {
588            int32_t what;
589            CHECK(msg->findInt32("what", &what));
590
591            switch (what) {
592                case CodecBase::kWhatError:
593                {
594                    int32_t omxError, internalError;
595                    CHECK(msg->findInt32("omx-error", &omxError));
596                    CHECK(msg->findInt32("err", &internalError));
597
598                    ALOGE("Codec reported an error. "
599                          "(omx error 0x%08x, internalError %d)",
600                          omxError, internalError);
601
602                    if (omxError == OMX_ErrorResourcesLost
603                            && internalError == DEAD_OBJECT) {
604                        mFlags |= kFlagSawMediaServerDie;
605                    }
606
607                    bool sendErrorReponse = true;
608
609                    switch (mState) {
610                        case INITIALIZING:
611                        {
612                            setState(UNINITIALIZED);
613                            break;
614                        }
615
616                        case CONFIGURING:
617                        {
618                            setState(INITIALIZED);
619                            break;
620                        }
621
622                        case STARTING:
623                        {
624                            setState(CONFIGURED);
625                            break;
626                        }
627
628                        case STOPPING:
629                        case RELEASING:
630                        {
631                            // Ignore the error, assuming we'll still get
632                            // the shutdown complete notification.
633
634                            sendErrorReponse = false;
635
636                            if (mFlags & kFlagSawMediaServerDie) {
637                                // MediaServer died, there definitely won't
638                                // be a shutdown complete notification after
639                                // all.
640
641                                // note that we're directly going from
642                                // STOPPING->UNINITIALIZED, instead of the
643                                // usual STOPPING->INITIALIZED state.
644                                setState(UNINITIALIZED);
645
646                                (new AMessage)->postReply(mReplyID);
647                            }
648                            break;
649                        }
650
651                        case FLUSHING:
652                        {
653                            setState(STARTED);
654                            break;
655                        }
656
657                        case STARTED:
658                        {
659                            sendErrorReponse = false;
660
661                            mFlags |= kFlagStickyError;
662                            postActivityNotificationIfPossible();
663
664                            cancelPendingDequeueOperations();
665
666                            if (mFlags & kFlagIsAsync) {
667                                onError(omxError, 0);
668                            }
669                            setState(UNINITIALIZED);
670                            break;
671                        }
672
673                        default:
674                        {
675                            sendErrorReponse = false;
676
677                            mFlags |= kFlagStickyError;
678                            postActivityNotificationIfPossible();
679
680                            if (mFlags & kFlagIsAsync) {
681                                onError(omxError, 0);
682                            }
683                            setState(UNINITIALIZED);
684                            break;
685                        }
686                    }
687
688                    if (sendErrorReponse) {
689                        PostReplyWithError(mReplyID, UNKNOWN_ERROR);
690                    }
691                    break;
692                }
693
694                case CodecBase::kWhatComponentAllocated:
695                {
696                    CHECK_EQ(mState, INITIALIZING);
697                    setState(INITIALIZED);
698
699                    CHECK(msg->findString("componentName", &mComponentName));
700
701                    if (mComponentName.startsWith("OMX.google.")) {
702                        mFlags |= kFlagIsSoftwareCodec;
703                    } else {
704                        mFlags &= ~kFlagIsSoftwareCodec;
705                    }
706
707                    if (mComponentName.endsWith(".secure")) {
708                        mFlags |= kFlagIsSecure;
709                    } else {
710                        mFlags &= ~kFlagIsSecure;
711                    }
712
713                    (new AMessage)->postReply(mReplyID);
714                    break;
715                }
716
717                case CodecBase::kWhatComponentConfigured:
718                {
719                    CHECK_EQ(mState, CONFIGURING);
720                    setState(CONFIGURED);
721
722                    // reset input surface flag
723                    mHaveInputSurface = false;
724
725                    CHECK(msg->findMessage("input-format", &mInputFormat));
726                    CHECK(msg->findMessage("output-format", &mOutputFormat));
727
728                    (new AMessage)->postReply(mReplyID);
729                    break;
730                }
731
732                case CodecBase::kWhatInputSurfaceCreated:
733                {
734                    // response to initiateCreateInputSurface()
735                    status_t err = NO_ERROR;
736                    sp<AMessage> response = new AMessage();
737                    if (!msg->findInt32("err", &err)) {
738                        sp<RefBase> obj;
739                        msg->findObject("input-surface", &obj);
740                        CHECK(obj != NULL);
741                        response->setObject("input-surface", obj);
742                        mHaveInputSurface = true;
743                    } else {
744                        response->setInt32("err", err);
745                    }
746                    response->postReply(mReplyID);
747                    break;
748                }
749
750                case CodecBase::kWhatSignaledInputEOS:
751                {
752                    // response to signalEndOfInputStream()
753                    sp<AMessage> response = new AMessage();
754                    status_t err;
755                    if (msg->findInt32("err", &err)) {
756                        response->setInt32("err", err);
757                    }
758                    response->postReply(mReplyID);
759                    break;
760                }
761
762
763                case CodecBase::kWhatBuffersAllocated:
764                {
765                    Mutex::Autolock al(mBufferLock);
766                    int32_t portIndex;
767                    CHECK(msg->findInt32("portIndex", &portIndex));
768
769                    ALOGV("%s buffers allocated",
770                          portIndex == kPortIndexInput ? "input" : "output");
771
772                    CHECK(portIndex == kPortIndexInput
773                            || portIndex == kPortIndexOutput);
774
775                    mPortBuffers[portIndex].clear();
776
777                    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
778
779                    sp<RefBase> obj;
780                    CHECK(msg->findObject("portDesc", &obj));
781
782                    sp<CodecBase::PortDescription> portDesc =
783                        static_cast<CodecBase::PortDescription *>(obj.get());
784
785                    size_t numBuffers = portDesc->countBuffers();
786
787                    for (size_t i = 0; i < numBuffers; ++i) {
788                        BufferInfo info;
789                        info.mBufferID = portDesc->bufferIDAt(i);
790                        info.mOwnedByClient = false;
791                        info.mData = portDesc->bufferAt(i);
792
793                        if (portIndex == kPortIndexInput && mCrypto != NULL) {
794                            info.mEncryptedData =
795                                new ABuffer(info.mData->capacity());
796                        }
797
798                        buffers->push_back(info);
799                    }
800
801                    if (portIndex == kPortIndexOutput) {
802                        if (mState == STARTING) {
803                            // We're always allocating output buffers after
804                            // allocating input buffers, so this is a good
805                            // indication that now all buffers are allocated.
806                            setState(STARTED);
807                            (new AMessage)->postReply(mReplyID);
808                        } else {
809                            mFlags |= kFlagOutputBuffersChanged;
810                            postActivityNotificationIfPossible();
811                        }
812                    }
813                    break;
814                }
815
816                case CodecBase::kWhatOutputFormatChanged:
817                {
818                    ALOGV("codec output format changed");
819
820                    if ((mFlags & kFlagIsSoftwareCodec)
821                            && mNativeWindow != NULL) {
822                        AString mime;
823                        CHECK(msg->findString("mime", &mime));
824
825                        if (!strncasecmp("video/", mime.c_str(), 6)) {
826                            delete mSoftRenderer;
827                            mSoftRenderer = NULL;
828
829                            int32_t width, height;
830                            CHECK(msg->findInt32("width", &width));
831                            CHECK(msg->findInt32("height", &height));
832
833                            int32_t cropLeft, cropTop, cropRight, cropBottom;
834                            CHECK(msg->findRect("crop",
835                                &cropLeft, &cropTop, &cropRight, &cropBottom));
836
837                            int32_t colorFormat;
838                            CHECK(msg->findInt32(
839                                        "color-format", &colorFormat));
840
841                            sp<MetaData> meta = new MetaData;
842                            meta->setInt32(kKeyWidth, width);
843                            meta->setInt32(kKeyHeight, height);
844                            meta->setRect(kKeyCropRect,
845                                cropLeft, cropTop, cropRight, cropBottom);
846                            meta->setInt32(kKeyColorFormat, colorFormat);
847
848                            mSoftRenderer =
849                                new SoftwareRenderer(mNativeWindow, meta);
850                        }
851                    }
852
853                    mOutputFormat = msg;
854
855                    if (mFlags & kFlagIsEncoder) {
856                        // Before we announce the format change we should
857                        // collect codec specific data and amend the output
858                        // format as necessary.
859                        mFlags |= kFlagGatherCodecSpecificData;
860                    } else if (mFlags & kFlagIsAsync) {
861                        onOutputFormatChanged();
862                    } else {
863                        mFlags |= kFlagOutputFormatChanged;
864                        postActivityNotificationIfPossible();
865                    }
866                    break;
867                }
868
869                case CodecBase::kWhatFillThisBuffer:
870                {
871                    /* size_t index = */updateBuffers(kPortIndexInput, msg);
872
873                    if (mState == FLUSHING
874                            || mState == STOPPING
875                            || mState == RELEASING) {
876                        returnBuffersToCodecOnPort(kPortIndexInput);
877                        break;
878                    }
879
880                    if (!mCSD.empty()) {
881                        ssize_t index = dequeuePortBuffer(kPortIndexInput);
882                        CHECK_GE(index, 0);
883
884                        // If codec specific data had been specified as
885                        // part of the format in the call to configure and
886                        // if there's more csd left, we submit it here
887                        // clients only get access to input buffers once
888                        // this data has been exhausted.
889
890                        status_t err = queueCSDInputBuffer(index);
891
892                        if (err != OK) {
893                            ALOGE("queueCSDInputBuffer failed w/ error %d",
894                                  err);
895
896                            mFlags |= kFlagStickyError;
897                            postActivityNotificationIfPossible();
898
899                            cancelPendingDequeueOperations();
900                        }
901                        break;
902                    }
903
904                    if (mFlags & kFlagIsAsync) {
905                        onInputBufferAvailable();
906                    } else if (mFlags & kFlagDequeueInputPending) {
907                        CHECK(handleDequeueInputBuffer(mDequeueInputReplyID));
908
909                        ++mDequeueInputTimeoutGeneration;
910                        mFlags &= ~kFlagDequeueInputPending;
911                        mDequeueInputReplyID = 0;
912                    } else {
913                        postActivityNotificationIfPossible();
914                    }
915                    break;
916                }
917
918                case CodecBase::kWhatDrainThisBuffer:
919                {
920                    /* size_t index = */updateBuffers(kPortIndexOutput, msg);
921
922                    if (mState == FLUSHING
923                            || mState == STOPPING
924                            || mState == RELEASING) {
925                        returnBuffersToCodecOnPort(kPortIndexOutput);
926                        break;
927                    }
928
929                    sp<ABuffer> buffer;
930                    CHECK(msg->findBuffer("buffer", &buffer));
931
932                    int32_t omxFlags;
933                    CHECK(msg->findInt32("flags", &omxFlags));
934
935                    buffer->meta()->setInt32("omxFlags", omxFlags);
936
937                    if (mFlags & kFlagGatherCodecSpecificData) {
938                        // This is the very first output buffer after a
939                        // format change was signalled, it'll either contain
940                        // the one piece of codec specific data we can expect
941                        // or there won't be codec specific data.
942                        if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) {
943                            status_t err =
944                                amendOutputFormatWithCodecSpecificData(buffer);
945
946                            if (err != OK) {
947                                ALOGE("Codec spit out malformed codec "
948                                      "specific data!");
949                            }
950                        }
951
952                        mFlags &= ~kFlagGatherCodecSpecificData;
953                        if (mFlags & kFlagIsAsync) {
954                            onOutputFormatChanged();
955                        } else {
956                            mFlags |= kFlagOutputFormatChanged;
957                        }
958                    }
959
960                    if (mFlags & kFlagIsAsync) {
961                        onOutputBufferAvailable();
962                    } else if (mFlags & kFlagDequeueOutputPending) {
963                        CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID));
964
965                        ++mDequeueOutputTimeoutGeneration;
966                        mFlags &= ~kFlagDequeueOutputPending;
967                        mDequeueOutputReplyID = 0;
968                    } else {
969                        postActivityNotificationIfPossible();
970                    }
971
972                    break;
973                }
974
975                case CodecBase::kWhatEOS:
976                {
977                    // We already notify the client of this by using the
978                    // corresponding flag in "onOutputBufferReady".
979                    break;
980                }
981
982                case CodecBase::kWhatShutdownCompleted:
983                {
984                    if (mState == STOPPING) {
985                        setState(INITIALIZED);
986                    } else {
987                        CHECK_EQ(mState, RELEASING);
988                        setState(UNINITIALIZED);
989                    }
990
991                    (new AMessage)->postReply(mReplyID);
992                    break;
993                }
994
995                case CodecBase::kWhatFlushCompleted:
996                {
997                    CHECK_EQ(mState, FLUSHING);
998                    setState(STARTED);
999
1000                    mCodec->signalResume();
1001
1002                    (new AMessage)->postReply(mReplyID);
1003                    break;
1004                }
1005
1006                default:
1007                    TRESPASS();
1008            }
1009            break;
1010        }
1011
1012        case kWhatInit:
1013        {
1014            uint32_t replyID;
1015            CHECK(msg->senderAwaitsResponse(&replyID));
1016
1017            if (mState != UNINITIALIZED) {
1018                PostReplyWithError(replyID, INVALID_OPERATION);
1019                break;
1020            }
1021
1022            mReplyID = replyID;
1023            setState(INITIALIZING);
1024
1025            AString name;
1026            CHECK(msg->findString("name", &name));
1027
1028            int32_t nameIsType;
1029            int32_t encoder = false;
1030            CHECK(msg->findInt32("nameIsType", &nameIsType));
1031            if (nameIsType) {
1032                CHECK(msg->findInt32("encoder", &encoder));
1033            }
1034
1035            sp<AMessage> format = new AMessage;
1036
1037            if (nameIsType) {
1038                format->setString("mime", name.c_str());
1039                format->setInt32("encoder", encoder);
1040            } else {
1041                format->setString("componentName", name.c_str());
1042            }
1043
1044            mCodec->initiateAllocateComponent(format);
1045            break;
1046        }
1047
1048        case kWhatSetCallback:
1049        {
1050            uint32_t replyID;
1051            CHECK(msg->senderAwaitsResponse(&replyID));
1052
1053            if (mState == UNINITIALIZED
1054                    || mState == INITIALIZING
1055                    || mState == STARTED) {
1056                // callback can't be set after codec is started,
1057                // or before it's initialized (as the callback
1058                // will be cleared when it goes to INITIALIZED)
1059                PostReplyWithError(replyID, INVALID_OPERATION);
1060                break;
1061            }
1062
1063            sp<AMessage> callback;
1064            CHECK(msg->findMessage("callback", &callback));
1065
1066            mCallback = callback;
1067
1068            if (mCallback != NULL) {
1069                ALOGI("MediaCodec will operate in async mode");
1070                mFlags |= kFlagIsAsync;
1071            } else {
1072                mFlags &= ~kFlagIsAsync;
1073            }
1074
1075            sp<AMessage> response = new AMessage;
1076            response->postReply(replyID);
1077            break;
1078        }
1079
1080        case kWhatConfigure:
1081        {
1082            uint32_t replyID;
1083            CHECK(msg->senderAwaitsResponse(&replyID));
1084
1085            if (mState != INITIALIZED) {
1086                PostReplyWithError(replyID, INVALID_OPERATION);
1087                break;
1088            }
1089
1090            sp<RefBase> obj;
1091            if (!msg->findObject("native-window", &obj)) {
1092                obj.clear();
1093            }
1094
1095            sp<AMessage> format;
1096            CHECK(msg->findMessage("format", &format));
1097
1098            if (obj != NULL) {
1099                format->setObject("native-window", obj);
1100
1101                status_t err = setNativeWindow(
1102                    static_cast<NativeWindowWrapper *>(obj.get())
1103                        ->getSurfaceTextureClient());
1104
1105                if (err != OK) {
1106                    PostReplyWithError(replyID, err);
1107                    break;
1108                }
1109            } else {
1110                setNativeWindow(NULL);
1111            }
1112
1113            mReplyID = replyID;
1114            setState(CONFIGURING);
1115
1116            void *crypto;
1117            if (!msg->findPointer("crypto", &crypto)) {
1118                crypto = NULL;
1119            }
1120
1121            mCrypto = static_cast<ICrypto *>(crypto);
1122
1123            uint32_t flags;
1124            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1125
1126            if (flags & CONFIGURE_FLAG_ENCODE) {
1127                format->setInt32("encoder", true);
1128                mFlags |= kFlagIsEncoder;
1129            }
1130
1131            extractCSD(format);
1132
1133            mCodec->initiateConfigureComponent(format);
1134            break;
1135        }
1136
1137        case kWhatCreateInputSurface:
1138        {
1139            uint32_t replyID;
1140            CHECK(msg->senderAwaitsResponse(&replyID));
1141
1142            // Must be configured, but can't have been started yet.
1143            if (mState != CONFIGURED) {
1144                PostReplyWithError(replyID, INVALID_OPERATION);
1145                break;
1146            }
1147
1148            mReplyID = replyID;
1149            mCodec->initiateCreateInputSurface();
1150            break;
1151        }
1152
1153        case kWhatStart:
1154        {
1155            uint32_t replyID;
1156            CHECK(msg->senderAwaitsResponse(&replyID));
1157
1158            if (mState != CONFIGURED) {
1159                PostReplyWithError(replyID, INVALID_OPERATION);
1160                break;
1161            }
1162
1163            mReplyID = replyID;
1164            setState(STARTING);
1165
1166            mCodec->initiateStart();
1167            break;
1168        }
1169
1170        case kWhatStop:
1171        case kWhatRelease:
1172        {
1173            State targetState =
1174                (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED;
1175
1176            uint32_t replyID;
1177            CHECK(msg->senderAwaitsResponse(&replyID));
1178
1179            if (mState != INITIALIZED
1180                    && mState != CONFIGURED && mState != STARTED) {
1181                // We may be in "UNINITIALIZED" state already without the
1182                // client being aware of this if media server died while
1183                // we were being stopped. The client would assume that
1184                // after stop() returned, it would be safe to call release()
1185                // and it should be in this case, no harm to allow a release()
1186                // if we're already uninitialized.
1187                // Similarly stopping a stopped MediaCodec should be benign.
1188                sp<AMessage> response = new AMessage;
1189                response->setInt32(
1190                        "err",
1191                        mState == targetState ? OK : INVALID_OPERATION);
1192
1193                response->postReply(replyID);
1194                break;
1195            }
1196
1197            if (mFlags & kFlagSawMediaServerDie) {
1198                // It's dead, Jim. Don't expect initiateShutdown to yield
1199                // any useful results now...
1200                setState(UNINITIALIZED);
1201                (new AMessage)->postReply(replyID);
1202                break;
1203            }
1204
1205            mReplyID = replyID;
1206            setState(msg->what() == kWhatStop ? STOPPING : RELEASING);
1207
1208            mCodec->initiateShutdown(
1209                    msg->what() == kWhatStop /* keepComponentAllocated */);
1210
1211            returnBuffersToCodec();
1212            break;
1213        }
1214
1215        case kWhatDequeueInputBuffer:
1216        {
1217            uint32_t replyID;
1218            CHECK(msg->senderAwaitsResponse(&replyID));
1219
1220            if (mFlags & kFlagIsAsync) {
1221                ALOGE("dequeueOutputBuffer can't be used in async mode");
1222                PostReplyWithError(replyID, INVALID_OPERATION);
1223                break;
1224            }
1225
1226            if (mHaveInputSurface) {
1227                ALOGE("dequeueInputBuffer can't be used with input surface");
1228                PostReplyWithError(replyID, INVALID_OPERATION);
1229                break;
1230            }
1231
1232            if (handleDequeueInputBuffer(replyID, true /* new request */)) {
1233                break;
1234            }
1235
1236            int64_t timeoutUs;
1237            CHECK(msg->findInt64("timeoutUs", &timeoutUs));
1238
1239            if (timeoutUs == 0ll) {
1240                PostReplyWithError(replyID, -EAGAIN);
1241                break;
1242            }
1243
1244            mFlags |= kFlagDequeueInputPending;
1245            mDequeueInputReplyID = replyID;
1246
1247            if (timeoutUs > 0ll) {
1248                sp<AMessage> timeoutMsg =
1249                    new AMessage(kWhatDequeueInputTimedOut, id());
1250                timeoutMsg->setInt32(
1251                        "generation", ++mDequeueInputTimeoutGeneration);
1252                timeoutMsg->post(timeoutUs);
1253            }
1254            break;
1255        }
1256
1257        case kWhatDequeueInputTimedOut:
1258        {
1259            int32_t generation;
1260            CHECK(msg->findInt32("generation", &generation));
1261
1262            if (generation != mDequeueInputTimeoutGeneration) {
1263                // Obsolete
1264                break;
1265            }
1266
1267            CHECK(mFlags & kFlagDequeueInputPending);
1268
1269            PostReplyWithError(mDequeueInputReplyID, -EAGAIN);
1270
1271            mFlags &= ~kFlagDequeueInputPending;
1272            mDequeueInputReplyID = 0;
1273            break;
1274        }
1275
1276        case kWhatQueueInputBuffer:
1277        {
1278            uint32_t replyID;
1279            CHECK(msg->senderAwaitsResponse(&replyID));
1280
1281            if (mState != STARTED || (mFlags & kFlagStickyError)) {
1282                PostReplyWithError(replyID, INVALID_OPERATION);
1283                break;
1284            }
1285
1286            status_t err = onQueueInputBuffer(msg);
1287
1288            PostReplyWithError(replyID, err);
1289            break;
1290        }
1291
1292        case kWhatDequeueOutputBuffer:
1293        {
1294            uint32_t replyID;
1295            CHECK(msg->senderAwaitsResponse(&replyID));
1296
1297            if (mFlags & kFlagIsAsync) {
1298                ALOGE("dequeueOutputBuffer can't be used in async mode");
1299                PostReplyWithError(replyID, INVALID_OPERATION);
1300                break;
1301            }
1302
1303            if (handleDequeueOutputBuffer(replyID, true /* new request */)) {
1304                break;
1305            }
1306
1307            int64_t timeoutUs;
1308            CHECK(msg->findInt64("timeoutUs", &timeoutUs));
1309
1310            if (timeoutUs == 0ll) {
1311                PostReplyWithError(replyID, -EAGAIN);
1312                break;
1313            }
1314
1315            mFlags |= kFlagDequeueOutputPending;
1316            mDequeueOutputReplyID = replyID;
1317
1318            if (timeoutUs > 0ll) {
1319                sp<AMessage> timeoutMsg =
1320                    new AMessage(kWhatDequeueOutputTimedOut, id());
1321                timeoutMsg->setInt32(
1322                        "generation", ++mDequeueOutputTimeoutGeneration);
1323                timeoutMsg->post(timeoutUs);
1324            }
1325            break;
1326        }
1327
1328        case kWhatDequeueOutputTimedOut:
1329        {
1330            int32_t generation;
1331            CHECK(msg->findInt32("generation", &generation));
1332
1333            if (generation != mDequeueOutputTimeoutGeneration) {
1334                // Obsolete
1335                break;
1336            }
1337
1338            CHECK(mFlags & kFlagDequeueOutputPending);
1339
1340            PostReplyWithError(mDequeueOutputReplyID, -EAGAIN);
1341
1342            mFlags &= ~kFlagDequeueOutputPending;
1343            mDequeueOutputReplyID = 0;
1344            break;
1345        }
1346
1347        case kWhatReleaseOutputBuffer:
1348        {
1349            uint32_t replyID;
1350            CHECK(msg->senderAwaitsResponse(&replyID));
1351
1352            if (mState != STARTED || (mFlags & kFlagStickyError)) {
1353                PostReplyWithError(replyID, INVALID_OPERATION);
1354                break;
1355            }
1356
1357            status_t err = onReleaseOutputBuffer(msg);
1358
1359            PostReplyWithError(replyID, err);
1360            break;
1361        }
1362
1363        case kWhatSignalEndOfInputStream:
1364        {
1365            uint32_t replyID;
1366            CHECK(msg->senderAwaitsResponse(&replyID));
1367
1368            if (mState != STARTED || (mFlags & kFlagStickyError)) {
1369                PostReplyWithError(replyID, INVALID_OPERATION);
1370                break;
1371            }
1372
1373            mReplyID = replyID;
1374            mCodec->signalEndOfInputStream();
1375            break;
1376        }
1377
1378        case kWhatGetBuffers:
1379        {
1380            uint32_t replyID;
1381            CHECK(msg->senderAwaitsResponse(&replyID));
1382
1383            if (mState != STARTED || (mFlags & kFlagStickyError)) {
1384                PostReplyWithError(replyID, INVALID_OPERATION);
1385                break;
1386            }
1387
1388            int32_t portIndex;
1389            CHECK(msg->findInt32("portIndex", &portIndex));
1390
1391            Vector<sp<ABuffer> > *dstBuffers;
1392            CHECK(msg->findPointer("buffers", (void **)&dstBuffers));
1393
1394            dstBuffers->clear();
1395            const Vector<BufferInfo> &srcBuffers = mPortBuffers[portIndex];
1396
1397            for (size_t i = 0; i < srcBuffers.size(); ++i) {
1398                const BufferInfo &info = srcBuffers.itemAt(i);
1399
1400                dstBuffers->push_back(
1401                        (portIndex == kPortIndexInput && mCrypto != NULL)
1402                                ? info.mEncryptedData : info.mData);
1403            }
1404
1405            (new AMessage)->postReply(replyID);
1406            break;
1407        }
1408
1409        case kWhatFlush:
1410        {
1411            uint32_t replyID;
1412            CHECK(msg->senderAwaitsResponse(&replyID));
1413
1414            if (mState != STARTED || (mFlags & kFlagStickyError)) {
1415                PostReplyWithError(replyID, INVALID_OPERATION);
1416                break;
1417            }
1418
1419            mReplyID = replyID;
1420            setState(FLUSHING);
1421
1422            mCodec->signalFlush();
1423            returnBuffersToCodec();
1424            break;
1425        }
1426
1427        case kWhatGetInputFormat:
1428        case kWhatGetOutputFormat:
1429        {
1430            sp<AMessage> format =
1431                (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat);
1432
1433            uint32_t replyID;
1434            CHECK(msg->senderAwaitsResponse(&replyID));
1435
1436            if ((mState != CONFIGURED && mState != STARTING &&
1437                 mState != STARTED && mState != FLUSHING)
1438                    || (mFlags & kFlagStickyError)
1439                    || format == NULL) {
1440                PostReplyWithError(replyID, INVALID_OPERATION);
1441                break;
1442            }
1443
1444            sp<AMessage> response = new AMessage;
1445            response->setMessage("format", format);
1446            response->postReply(replyID);
1447            break;
1448        }
1449
1450        case kWhatRequestIDRFrame:
1451        {
1452            mCodec->signalRequestIDRFrame();
1453            break;
1454        }
1455
1456        case kWhatRequestActivityNotification:
1457        {
1458            CHECK(mActivityNotify == NULL);
1459            CHECK(msg->findMessage("notify", &mActivityNotify));
1460
1461            postActivityNotificationIfPossible();
1462            break;
1463        }
1464
1465        case kWhatGetName:
1466        {
1467            uint32_t replyID;
1468            CHECK(msg->senderAwaitsResponse(&replyID));
1469
1470            if (mComponentName.empty()) {
1471                PostReplyWithError(replyID, INVALID_OPERATION);
1472                break;
1473            }
1474
1475            sp<AMessage> response = new AMessage;
1476            response->setString("name", mComponentName.c_str());
1477            response->postReply(replyID);
1478            break;
1479        }
1480
1481        case kWhatSetParameters:
1482        {
1483            uint32_t replyID;
1484            CHECK(msg->senderAwaitsResponse(&replyID));
1485
1486            sp<AMessage> params;
1487            CHECK(msg->findMessage("params", &params));
1488
1489            status_t err = onSetParameters(params);
1490
1491            PostReplyWithError(replyID, err);
1492            break;
1493        }
1494
1495        default:
1496            TRESPASS();
1497    }
1498}
1499
1500void MediaCodec::extractCSD(const sp<AMessage> &format) {
1501    mCSD.clear();
1502
1503    size_t i = 0;
1504    for (;;) {
1505        sp<ABuffer> csd;
1506        if (!format->findBuffer(StringPrintf("csd-%u", i).c_str(), &csd)) {
1507            break;
1508        }
1509
1510        mCSD.push_back(csd);
1511        ++i;
1512    }
1513
1514    ALOGV("Found %zu pieces of codec specific data.", mCSD.size());
1515}
1516
1517status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) {
1518    CHECK(!mCSD.empty());
1519
1520    const BufferInfo *info =
1521        &mPortBuffers[kPortIndexInput].itemAt(bufferIndex);
1522
1523    sp<ABuffer> csd = *mCSD.begin();
1524    mCSD.erase(mCSD.begin());
1525
1526    const sp<ABuffer> &codecInputData =
1527        (mCrypto != NULL) ? info->mEncryptedData : info->mData;
1528
1529    if (csd->size() > codecInputData->capacity()) {
1530        return -EINVAL;
1531    }
1532
1533    memcpy(codecInputData->data(), csd->data(), csd->size());
1534
1535    AString errorDetailMsg;
1536
1537    sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, id());
1538    msg->setSize("index", bufferIndex);
1539    msg->setSize("offset", 0);
1540    msg->setSize("size", csd->size());
1541    msg->setInt64("timeUs", 0ll);
1542    msg->setInt32("flags", BUFFER_FLAG_CODECCONFIG);
1543    msg->setPointer("errorDetailMsg", &errorDetailMsg);
1544
1545    return onQueueInputBuffer(msg);
1546}
1547
1548void MediaCodec::setState(State newState) {
1549    if (newState == INITIALIZED || newState == UNINITIALIZED) {
1550        delete mSoftRenderer;
1551        mSoftRenderer = NULL;
1552
1553        mCrypto.clear();
1554        setNativeWindow(NULL);
1555
1556        mOutputFormat.clear();
1557        mFlags &= ~kFlagOutputFormatChanged;
1558        mFlags &= ~kFlagOutputBuffersChanged;
1559        mFlags &= ~kFlagStickyError;
1560        mFlags &= ~kFlagIsEncoder;
1561        mFlags &= ~kFlagGatherCodecSpecificData;
1562        mFlags &= ~kFlagIsAsync;
1563
1564        mActivityNotify.clear();
1565        mCallback.clear();
1566    }
1567
1568    if (newState == UNINITIALIZED) {
1569        mComponentName.clear();
1570
1571        // The component is gone, mediaserver's probably back up already
1572        // but should definitely be back up should we try to instantiate
1573        // another component.. and the cycle continues.
1574        mFlags &= ~kFlagSawMediaServerDie;
1575    }
1576
1577    mState = newState;
1578
1579    cancelPendingDequeueOperations();
1580}
1581
1582void MediaCodec::returnBuffersToCodec() {
1583    returnBuffersToCodecOnPort(kPortIndexInput);
1584    returnBuffersToCodecOnPort(kPortIndexOutput);
1585}
1586
1587void MediaCodec::returnBuffersToCodecOnPort(int32_t portIndex) {
1588    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
1589    Mutex::Autolock al(mBufferLock);
1590
1591    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
1592
1593    for (size_t i = 0; i < buffers->size(); ++i) {
1594        BufferInfo *info = &buffers->editItemAt(i);
1595
1596        if (info->mNotify != NULL) {
1597            sp<AMessage> msg = info->mNotify;
1598            info->mNotify = NULL;
1599            info->mOwnedByClient = false;
1600
1601            if (portIndex == kPortIndexInput) {
1602                /* no error, just returning buffers */
1603                msg->setInt32("err", OK);
1604            }
1605            msg->post();
1606        }
1607    }
1608
1609    mAvailPortBuffers[portIndex].clear();
1610}
1611
1612size_t MediaCodec::updateBuffers(
1613        int32_t portIndex, const sp<AMessage> &msg) {
1614    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
1615
1616    uint32_t bufferID;
1617    CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID));
1618
1619    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
1620
1621    for (size_t i = 0; i < buffers->size(); ++i) {
1622        BufferInfo *info = &buffers->editItemAt(i);
1623
1624        if (info->mBufferID == bufferID) {
1625            CHECK(info->mNotify == NULL);
1626            CHECK(msg->findMessage("reply", &info->mNotify));
1627
1628            mAvailPortBuffers[portIndex].push_back(i);
1629
1630            return i;
1631        }
1632    }
1633
1634    TRESPASS();
1635
1636    return 0;
1637}
1638
1639status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) {
1640    size_t index;
1641    size_t offset;
1642    size_t size;
1643    int64_t timeUs;
1644    uint32_t flags;
1645    CHECK(msg->findSize("index", &index));
1646    CHECK(msg->findSize("offset", &offset));
1647    CHECK(msg->findInt64("timeUs", &timeUs));
1648    CHECK(msg->findInt32("flags", (int32_t *)&flags));
1649
1650    const CryptoPlugin::SubSample *subSamples;
1651    size_t numSubSamples;
1652    const uint8_t *key;
1653    const uint8_t *iv;
1654    CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted;
1655
1656    // We allow the simpler queueInputBuffer API to be used even in
1657    // secure mode, by fabricating a single unencrypted subSample.
1658    CryptoPlugin::SubSample ss;
1659
1660    if (msg->findSize("size", &size)) {
1661        if (mCrypto != NULL) {
1662            ss.mNumBytesOfClearData = size;
1663            ss.mNumBytesOfEncryptedData = 0;
1664
1665            subSamples = &ss;
1666            numSubSamples = 1;
1667            key = NULL;
1668            iv = NULL;
1669        }
1670    } else {
1671        if (mCrypto == NULL) {
1672            return -EINVAL;
1673        }
1674
1675        CHECK(msg->findPointer("subSamples", (void **)&subSamples));
1676        CHECK(msg->findSize("numSubSamples", &numSubSamples));
1677        CHECK(msg->findPointer("key", (void **)&key));
1678        CHECK(msg->findPointer("iv", (void **)&iv));
1679
1680        int32_t tmp;
1681        CHECK(msg->findInt32("mode", &tmp));
1682
1683        mode = (CryptoPlugin::Mode)tmp;
1684
1685        size = 0;
1686        for (size_t i = 0; i < numSubSamples; ++i) {
1687            size += subSamples[i].mNumBytesOfClearData;
1688            size += subSamples[i].mNumBytesOfEncryptedData;
1689        }
1690    }
1691
1692    if (index >= mPortBuffers[kPortIndexInput].size()) {
1693        return -ERANGE;
1694    }
1695
1696    BufferInfo *info = &mPortBuffers[kPortIndexInput].editItemAt(index);
1697
1698    if (info->mNotify == NULL || !info->mOwnedByClient) {
1699        return -EACCES;
1700    }
1701
1702    if (offset + size > info->mData->capacity()) {
1703        return -EINVAL;
1704    }
1705
1706    sp<AMessage> reply = info->mNotify;
1707    info->mData->setRange(offset, size);
1708    info->mData->meta()->setInt64("timeUs", timeUs);
1709
1710    if (flags & BUFFER_FLAG_EOS) {
1711        info->mData->meta()->setInt32("eos", true);
1712    }
1713
1714    if (flags & BUFFER_FLAG_CODECCONFIG) {
1715        info->mData->meta()->setInt32("csd", true);
1716    }
1717
1718    if (mCrypto != NULL) {
1719        if (size > info->mEncryptedData->capacity()) {
1720            return -ERANGE;
1721        }
1722
1723        AString *errorDetailMsg;
1724        CHECK(msg->findPointer("errorDetailMsg", (void **)&errorDetailMsg));
1725
1726        ssize_t result = mCrypto->decrypt(
1727                (mFlags & kFlagIsSecure) != 0,
1728                key,
1729                iv,
1730                mode,
1731                info->mEncryptedData->base() + offset,
1732                subSamples,
1733                numSubSamples,
1734                info->mData->base(),
1735                errorDetailMsg);
1736
1737        if (result < 0) {
1738            return result;
1739        }
1740
1741        info->mData->setRange(0, result);
1742    }
1743
1744    // synchronization boundary for getBufferAndFormat
1745    {
1746        Mutex::Autolock al(mBufferLock);
1747        info->mOwnedByClient = false;
1748    }
1749    reply->setBuffer("buffer", info->mData);
1750    reply->post();
1751
1752    info->mNotify = NULL;
1753
1754    return OK;
1755}
1756
1757status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
1758    size_t index;
1759    CHECK(msg->findSize("index", &index));
1760
1761    int32_t render;
1762    if (!msg->findInt32("render", &render)) {
1763        render = 0;
1764    }
1765
1766    if (mState != STARTED) {
1767        return -EINVAL;
1768    }
1769
1770    if (index >= mPortBuffers[kPortIndexOutput].size()) {
1771        return -ERANGE;
1772    }
1773
1774    BufferInfo *info = &mPortBuffers[kPortIndexOutput].editItemAt(index);
1775
1776    if (info->mNotify == NULL || !info->mOwnedByClient) {
1777        return -EACCES;
1778    }
1779
1780    // synchronization boundary for getBufferAndFormat
1781    {
1782        Mutex::Autolock al(mBufferLock);
1783        info->mOwnedByClient = false;
1784    }
1785
1786    if (render && info->mData != NULL && info->mData->size() != 0) {
1787        info->mNotify->setInt32("render", true);
1788
1789        int64_t timestampNs = 0;
1790        if (msg->findInt64("timestampNs", &timestampNs)) {
1791            info->mNotify->setInt64("timestampNs", timestampNs);
1792        } else {
1793            // TODO: it seems like we should use the timestamp
1794            // in the (media)buffer as it potentially came from
1795            // an input surface, but we did not propagate it prior to
1796            // API 20.  Perhaps check for target SDK version.
1797#if 0
1798            if (info->mData->meta()->findInt64("timeUs", &timestampNs)) {
1799                ALOGV("using buffer PTS of %" PRId64, timestampNs);
1800                timestampNs *= 1000;
1801            }
1802#endif
1803        }
1804
1805        if (mSoftRenderer != NULL) {
1806            mSoftRenderer->render(
1807                    info->mData->data(), info->mData->size(), timestampNs, NULL);
1808        }
1809    }
1810
1811    info->mNotify->post();
1812    info->mNotify = NULL;
1813
1814    return OK;
1815}
1816
1817ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {
1818    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
1819
1820    List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];
1821
1822    if (availBuffers->empty()) {
1823        return -EAGAIN;
1824    }
1825
1826    size_t index = *availBuffers->begin();
1827    availBuffers->erase(availBuffers->begin());
1828
1829    BufferInfo *info = &mPortBuffers[portIndex].editItemAt(index);
1830    CHECK(!info->mOwnedByClient);
1831    {
1832        Mutex::Autolock al(mBufferLock);
1833        info->mFormat = portIndex == kPortIndexInput ? mInputFormat : mOutputFormat;
1834        info->mOwnedByClient = true;
1835    }
1836
1837    return index;
1838}
1839
1840status_t MediaCodec::setNativeWindow(
1841        const sp<Surface> &surfaceTextureClient) {
1842    status_t err;
1843
1844    if (mNativeWindow != NULL) {
1845        err = native_window_api_disconnect(
1846                mNativeWindow.get(), NATIVE_WINDOW_API_MEDIA);
1847
1848        if (err != OK) {
1849            ALOGW("native_window_api_disconnect returned an error: %s (%d)",
1850                    strerror(-err), err);
1851        }
1852
1853        mNativeWindow.clear();
1854    }
1855
1856    if (surfaceTextureClient != NULL) {
1857        err = native_window_api_connect(
1858                surfaceTextureClient.get(), NATIVE_WINDOW_API_MEDIA);
1859
1860        if (err != OK) {
1861            ALOGE("native_window_api_connect returned an error: %s (%d)",
1862                    strerror(-err), err);
1863
1864            return err;
1865        }
1866
1867        mNativeWindow = surfaceTextureClient;
1868    }
1869
1870    return OK;
1871}
1872
1873void MediaCodec::onInputBufferAvailable() {
1874    int32_t index;
1875    while ((index = dequeuePortBuffer(kPortIndexInput)) >= 0) {
1876        sp<AMessage> msg = mCallback->dup();
1877        msg->setInt32("callbackID", CB_INPUT_AVAILABLE);
1878        msg->setInt32("index", index);
1879        msg->post();
1880    }
1881}
1882
1883void MediaCodec::onOutputBufferAvailable() {
1884    int32_t index;
1885    while ((index = dequeuePortBuffer(kPortIndexOutput)) >= 0) {
1886        const sp<ABuffer> &buffer =
1887            mPortBuffers[kPortIndexOutput].itemAt(index).mData;
1888        sp<AMessage> msg = mCallback->dup();
1889        msg->setInt32("callbackID", CB_OUTPUT_AVAILABLE);
1890        msg->setInt32("index", index);
1891        msg->setSize("offset", buffer->offset());
1892        msg->setSize("size", buffer->size());
1893
1894        int64_t timeUs;
1895        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1896
1897        msg->setInt64("timeUs", timeUs);
1898
1899        int32_t omxFlags;
1900        CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags));
1901
1902        uint32_t flags = 0;
1903        if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) {
1904            flags |= BUFFER_FLAG_SYNCFRAME;
1905        }
1906        if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) {
1907            flags |= BUFFER_FLAG_CODECCONFIG;
1908        }
1909        if (omxFlags & OMX_BUFFERFLAG_EOS) {
1910            flags |= BUFFER_FLAG_EOS;
1911        }
1912
1913        msg->setInt32("flags", flags);
1914
1915        msg->post();
1916    }
1917}
1918
1919void MediaCodec::onError(status_t err, int32_t actionCode, const char *detail) {
1920    if (mCallback != NULL) {
1921        sp<AMessage> msg = mCallback->dup();
1922        msg->setInt32("callbackID", CB_ERROR);
1923        msg->setInt32("err", err);
1924        msg->setInt32("actionCode", actionCode);
1925
1926        if (detail != NULL) {
1927            msg->setString("detail", detail);
1928        }
1929
1930        msg->post();
1931    }
1932}
1933
1934void MediaCodec::onOutputFormatChanged() {
1935    if (mCallback != NULL) {
1936        sp<AMessage> msg = mCallback->dup();
1937        msg->setInt32("callbackID", CB_OUTPUT_FORMAT_CHANGED);
1938        msg->setMessage("format", mOutputFormat);
1939        msg->post();
1940    }
1941}
1942
1943
1944void MediaCodec::postActivityNotificationIfPossible() {
1945    if (mActivityNotify == NULL) {
1946        return;
1947    }
1948
1949    if ((mFlags & (kFlagStickyError
1950                    | kFlagOutputBuffersChanged
1951                    | kFlagOutputFormatChanged))
1952            || !mAvailPortBuffers[kPortIndexInput].empty()
1953            || !mAvailPortBuffers[kPortIndexOutput].empty()) {
1954        mActivityNotify->post();
1955        mActivityNotify.clear();
1956    }
1957}
1958
1959status_t MediaCodec::setParameters(const sp<AMessage> &params) {
1960    sp<AMessage> msg = new AMessage(kWhatSetParameters, id());
1961    msg->setMessage("params", params);
1962
1963    sp<AMessage> response;
1964    return PostAndAwaitResponse(msg, &response);
1965}
1966
1967status_t MediaCodec::onSetParameters(const sp<AMessage> &params) {
1968    mCodec->signalSetParameters(params);
1969
1970    return OK;
1971}
1972
1973status_t MediaCodec::amendOutputFormatWithCodecSpecificData(
1974        const sp<ABuffer> &buffer) {
1975    AString mime;
1976    CHECK(mOutputFormat->findString("mime", &mime));
1977
1978    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) {
1979        // Codec specific data should be SPS and PPS in a single buffer,
1980        // each prefixed by a startcode (0x00 0x00 0x00 0x01).
1981        // We separate the two and put them into the output format
1982        // under the keys "csd-0" and "csd-1".
1983
1984        unsigned csdIndex = 0;
1985
1986        const uint8_t *data = buffer->data();
1987        size_t size = buffer->size();
1988
1989        const uint8_t *nalStart;
1990        size_t nalSize;
1991        while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) {
1992            sp<ABuffer> csd = new ABuffer(nalSize + 4);
1993            memcpy(csd->data(), "\x00\x00\x00\x01", 4);
1994            memcpy(csd->data() + 4, nalStart, nalSize);
1995
1996            mOutputFormat->setBuffer(
1997                    StringPrintf("csd-%u", csdIndex).c_str(), csd);
1998
1999            ++csdIndex;
2000        }
2001
2002        if (csdIndex != 2) {
2003            return ERROR_MALFORMED;
2004        }
2005    } else {
2006        // For everything else we just stash the codec specific data into
2007        // the output format as a single piece of csd under "csd-0".
2008        mOutputFormat->setBuffer("csd-0", buffer);
2009    }
2010
2011    return OK;
2012}
2013
2014}  // namespace android
2015