MediaCodec.cpp revision 0e8cfc36044ba97545e7c9e129b0b3e98eec5089 
1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "MediaCodec"
19#include <inttypes.h>
20
21#include "include/avc_utils.h"
22#include "include/SoftwareRenderer.h"
23
24#include <binder/IBatteryStats.h>
25#include <binder/IServiceManager.h>
26#include <gui/Surface.h>
27#include <media/ICrypto.h>
28#include <media/stagefright/foundation/ABuffer.h>
29#include <media/stagefright/foundation/ADebug.h>
30#include <media/stagefright/foundation/AMessage.h>
31#include <media/stagefright/foundation/AString.h>
32#include <media/stagefright/foundation/hexdump.h>
33#include <media/stagefright/ACodec.h>
34#include <media/stagefright/BufferProducerWrapper.h>
35#include <media/stagefright/MediaCodec.h>
36#include <media/stagefright/MediaCodecList.h>
37#include <media/stagefright/MediaDefs.h>
38#include <media/stagefright/MediaErrors.h>
39#include <media/stagefright/MetaData.h>
40#include <media/stagefright/NativeWindowWrapper.h>
41#include <private/android_filesystem_config.h>
42#include <utils/Log.h>
43#include <utils/Singleton.h>
44
45namespace android {
46
47struct MediaCodec::BatteryNotifier : public Singleton<BatteryNotifier> {
48    BatteryNotifier();
49
50    void noteStartVideo();
51    void noteStopVideo();
52    void noteStartAudio();
53    void noteStopAudio();
54
55private:
56    int32_t mVideoRefCount;
57    int32_t mAudioRefCount;
58    sp<IBatteryStats> mBatteryStatService;
59};
60
61ANDROID_SINGLETON_STATIC_INSTANCE(MediaCodec::BatteryNotifier)
62
63MediaCodec::BatteryNotifier::BatteryNotifier() :
64    mVideoRefCount(0),
65    mAudioRefCount(0) {
66    // get battery service
67    const sp<IServiceManager> sm(defaultServiceManager());
68    if (sm != NULL) {
69        const String16 name("batterystats");
70        mBatteryStatService = interface_cast<IBatteryStats>(sm->getService(name));
71        if (mBatteryStatService == NULL) {
72            ALOGE("batterystats service unavailable!");
73        }
74    }
75}
76
77void MediaCodec::BatteryNotifier::noteStartVideo() {
78    if (mVideoRefCount == 0 && mBatteryStatService != NULL) {
79        mBatteryStatService->noteStartVideo(AID_MEDIA);
80    }
81    mVideoRefCount++;
82}
83
84void MediaCodec::BatteryNotifier::noteStopVideo() {
85    if (mVideoRefCount == 0) {
86        ALOGW("BatteryNotifier::noteStop(): video refcount is broken!");
87        return;
88    }
89
90    mVideoRefCount--;
91    if (mVideoRefCount == 0 && mBatteryStatService != NULL) {
92        mBatteryStatService->noteStopVideo(AID_MEDIA);
93    }
94}
95
96void MediaCodec::BatteryNotifier::noteStartAudio() {
97    if (mAudioRefCount == 0 && mBatteryStatService != NULL) {
98        mBatteryStatService->noteStartAudio(AID_MEDIA);
99    }
100    mAudioRefCount++;
101}
102
103void MediaCodec::BatteryNotifier::noteStopAudio() {
104    if (mAudioRefCount == 0) {
105        ALOGW("BatteryNotifier::noteStop(): audio refcount is broken!");
106        return;
107    }
108
109    mAudioRefCount--;
110    if (mAudioRefCount == 0 && mBatteryStatService != NULL) {
111        mBatteryStatService->noteStopAudio(AID_MEDIA);
112    }
113}
114// static
115sp<MediaCodec> MediaCodec::CreateByType(
116        const sp<ALooper> &looper, const char *mime, bool encoder) {
117    sp<MediaCodec> codec = new MediaCodec(looper);
118    if (codec->init(mime, true /* nameIsType */, encoder) != OK) {
119        return NULL;
120    }
121
122    return codec;
123}
124
125// static
126sp<MediaCodec> MediaCodec::CreateByComponentName(
127        const sp<ALooper> &looper, const char *name) {
128    sp<MediaCodec> codec = new MediaCodec(looper);
129    if (codec->init(name, false /* nameIsType */, false /* encoder */) != OK) {
130        return NULL;
131    }
132
133    return codec;
134}
135
136MediaCodec::MediaCodec(const sp<ALooper> &looper)
137    : mState(UNINITIALIZED),
138      mLooper(looper),
139      mCodec(NULL),
140      mReplyID(0),
141      mFlags(0),
142      mSoftRenderer(NULL),
143      mBatteryStatNotified(false),
144      mIsVideo(false),
145      mDequeueInputTimeoutGeneration(0),
146      mDequeueInputReplyID(0),
147      mDequeueOutputTimeoutGeneration(0),
148      mDequeueOutputReplyID(0),
149      mHaveInputSurface(false) {
150}
151
152MediaCodec::~MediaCodec() {
153    CHECK_EQ(mState, UNINITIALIZED);
154}
155
156// static
157status_t MediaCodec::PostAndAwaitResponse(
158        const sp<AMessage> &msg, sp<AMessage> *response) {
159    status_t err = msg->postAndAwaitResponse(response);
160
161    if (err != OK) {
162        return err;
163    }
164
165    if (!(*response)->findInt32("err", &err)) {
166        err = OK;
167    }
168
169    return err;
170}
171
172// static
173void MediaCodec::PostReplyWithError(int32_t replyID, int32_t err) {
174    sp<AMessage> response = new AMessage;
175    response->setInt32("err", err);
176    response->postReply(replyID);
177}
178
179status_t MediaCodec::init(const char *name, bool nameIsType, bool encoder) {
180    // save init parameters for reset
181    mInitName = name;
182    mInitNameIsType = nameIsType;
183    mInitIsEncoder = encoder;
184
185    // Current video decoders do not return from OMX_FillThisBuffer
186    // quickly, violating the OpenMAX specs, until that is remedied
187    // we need to invest in an extra looper to free the main event
188    // queue.
189    mCodec = new ACodec;
190    bool needDedicatedLooper = false;
191    if (nameIsType && !strncasecmp(name, "video/", 6)) {
192        needDedicatedLooper = true;
193    } else {
194        AString tmp = name;
195        if (tmp.endsWith(".secure")) {
196            tmp.erase(tmp.size() - 7, 7);
197        }
198        const MediaCodecList *mcl = MediaCodecList::getInstance();
199        ssize_t codecIdx = mcl->findCodecByName(tmp.c_str());
200        if (codecIdx >= 0) {
201            Vector<AString> types;
202            if (mcl->getSupportedTypes(codecIdx, &types) == OK) {
203                for (size_t i = 0; i < types.size(); i++) {
204                    if (types[i].startsWith("video/")) {
205                        needDedicatedLooper = true;
206                        break;
207                    }
208                }
209            }
210        }
211    }
212
213    if (needDedicatedLooper) {
214        if (mCodecLooper == NULL) {
215            mCodecLooper = new ALooper;
216            mCodecLooper->setName("CodecLooper");
217            mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
218        }
219
220        mCodecLooper->registerHandler(mCodec);
221    } else {
222        mLooper->registerHandler(mCodec);
223    }
224
225    mLooper->registerHandler(this);
226
227    mCodec->setNotificationMessage(new AMessage(kWhatCodecNotify, id()));
228
229    sp<AMessage> msg = new AMessage(kWhatInit, id());
230    msg->setString("name", name);
231    msg->setInt32("nameIsType", nameIsType);
232
233    if (nameIsType) {
234        msg->setInt32("encoder", encoder);
235    }
236
237    sp<AMessage> response;
238    return PostAndAwaitResponse(msg, &response);
239}
240
241status_t MediaCodec::setCallback(const sp<AMessage> &callback) {
242    sp<AMessage> msg = new AMessage(kWhatSetCallback, id());
243    msg->setMessage("callback", callback);
244
245    sp<AMessage> response;
246    return PostAndAwaitResponse(msg, &response);
247}
248
249status_t MediaCodec::configure(
250        const sp<AMessage> &format,
251        const sp<Surface> &nativeWindow,
252        const sp<ICrypto> &crypto,
253        uint32_t flags) {
254    sp<AMessage> msg = new AMessage(kWhatConfigure, id());
255
256    msg->setMessage("format", format);
257    msg->setInt32("flags", flags);
258
259    if (nativeWindow != NULL) {
260        msg->setObject(
261                "native-window",
262                new NativeWindowWrapper(nativeWindow));
263    }
264
265    if (crypto != NULL) {
266        msg->setPointer("crypto", crypto.get());
267    }
268
269    sp<AMessage> response;
270    return PostAndAwaitResponse(msg, &response);
271}
272
273status_t MediaCodec::createInputSurface(
274        sp<IGraphicBufferProducer>* bufferProducer) {
275    sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, id());
276
277    sp<AMessage> response;
278    status_t err = PostAndAwaitResponse(msg, &response);
279    if (err == NO_ERROR) {
280        // unwrap the sp<IGraphicBufferProducer>
281        sp<RefBase> obj;
282        bool found = response->findObject("input-surface", &obj);
283        CHECK(found);
284        sp<BufferProducerWrapper> wrapper(
285                static_cast<BufferProducerWrapper*>(obj.get()));
286        *bufferProducer = wrapper->getBufferProducer();
287    } else {
288        ALOGW("createInputSurface failed, err=%d", err);
289    }
290    return err;
291}
292
293status_t MediaCodec::start() {
294    sp<AMessage> msg = new AMessage(kWhatStart, id());
295
296    sp<AMessage> response;
297    return PostAndAwaitResponse(msg, &response);
298}
299
300status_t MediaCodec::stop() {
301    sp<AMessage> msg = new AMessage(kWhatStop, id());
302
303    sp<AMessage> response;
304    return PostAndAwaitResponse(msg, &response);
305}
306
307status_t MediaCodec::release() {
308    sp<AMessage> msg = new AMessage(kWhatRelease, id());
309
310    sp<AMessage> response;
311    return PostAndAwaitResponse(msg, &response);
312}
313
314status_t MediaCodec::reset() {
315    /* When external-facing MediaCodec object is created,
316       it is already initialized.  Thus, reset is essentially
317       release() followed by init(), plus clearing the state */
318
319    status_t err = release();
320
321    // unregister handlers
322    if (mCodec != NULL) {
323        if (mCodecLooper != NULL) {
324            mCodecLooper->unregisterHandler(mCodec->id());
325        } else {
326            mLooper->unregisterHandler(mCodec->id());
327        }
328        mCodec = NULL;
329    }
330    mLooper->unregisterHandler(id());
331
332    mFlags = 0;    // clear all flags
333
334    // reset state not reset by setState(UNINITIALIZED)
335    mReplyID = 0;
336    mDequeueInputReplyID = 0;
337    mDequeueOutputReplyID = 0;
338    mDequeueInputTimeoutGeneration = 0;
339    mDequeueOutputTimeoutGeneration = 0;
340    mHaveInputSurface = false;
341
342    if (err == OK) {
343        err = init(mInitName.c_str(), mInitNameIsType, mInitIsEncoder);
344    }
345    return err;
346}
347
348status_t MediaCodec::queueInputBuffer(
349        size_t index,
350        size_t offset,
351        size_t size,
352        int64_t presentationTimeUs,
353        uint32_t flags,
354        AString *errorDetailMsg) {
355    if (errorDetailMsg != NULL) {
356        errorDetailMsg->clear();
357    }
358
359    sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, id());
360    msg->setSize("index", index);
361    msg->setSize("offset", offset);
362    msg->setSize("size", size);
363    msg->setInt64("timeUs", presentationTimeUs);
364    msg->setInt32("flags", flags);
365    msg->setPointer("errorDetailMsg", errorDetailMsg);
366
367    sp<AMessage> response;
368    return PostAndAwaitResponse(msg, &response);
369}
370
371status_t MediaCodec::queueSecureInputBuffer(
372        size_t index,
373        size_t offset,
374        const CryptoPlugin::SubSample *subSamples,
375        size_t numSubSamples,
376        const uint8_t key[16],
377        const uint8_t iv[16],
378        CryptoPlugin::Mode mode,
379        int64_t presentationTimeUs,
380        uint32_t flags,
381        AString *errorDetailMsg) {
382    if (errorDetailMsg != NULL) {
383        errorDetailMsg->clear();
384    }
385
386    sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, id());
387    msg->setSize("index", index);
388    msg->setSize("offset", offset);
389    msg->setPointer("subSamples", (void *)subSamples);
390    msg->setSize("numSubSamples", numSubSamples);
391    msg->setPointer("key", (void *)key);
392    msg->setPointer("iv", (void *)iv);
393    msg->setInt32("mode", mode);
394    msg->setInt64("timeUs", presentationTimeUs);
395    msg->setInt32("flags", flags);
396    msg->setPointer("errorDetailMsg", errorDetailMsg);
397
398    sp<AMessage> response;
399    status_t err = PostAndAwaitResponse(msg, &response);
400
401    return err;
402}
403
404status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) {
405    sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, id());
406    msg->setInt64("timeoutUs", timeoutUs);
407
408    sp<AMessage> response;
409    status_t err;
410    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
411        return err;
412    }
413
414    CHECK(response->findSize("index", index));
415
416    return OK;
417}
418
419status_t MediaCodec::dequeueOutputBuffer(
420        size_t *index,
421        size_t *offset,
422        size_t *size,
423        int64_t *presentationTimeUs,
424        uint32_t *flags,
425        int64_t timeoutUs) {
426    sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, id());
427    msg->setInt64("timeoutUs", timeoutUs);
428
429    sp<AMessage> response;
430    status_t err;
431    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
432        return err;
433    }
434
435    CHECK(response->findSize("index", index));
436    CHECK(response->findSize("offset", offset));
437    CHECK(response->findSize("size", size));
438    CHECK(response->findInt64("timeUs", presentationTimeUs));
439    CHECK(response->findInt32("flags", (int32_t *)flags));
440
441    return OK;
442}
443
444status_t MediaCodec::renderOutputBufferAndRelease(size_t index) {
445    sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, id());
446    msg->setSize("index", index);
447    msg->setInt32("render", true);
448
449    sp<AMessage> response;
450    return PostAndAwaitResponse(msg, &response);
451}
452
453status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) {
454    sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, id());
455    msg->setSize("index", index);
456    msg->setInt32("render", true);
457    msg->setInt64("timestampNs", timestampNs);
458
459    sp<AMessage> response;
460    return PostAndAwaitResponse(msg, &response);
461}
462
463status_t MediaCodec::releaseOutputBuffer(size_t index) {
464    sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, id());
465    msg->setSize("index", index);
466
467    sp<AMessage> response;
468    return PostAndAwaitResponse(msg, &response);
469}
470
471status_t MediaCodec::signalEndOfInputStream() {
472    sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, id());
473
474    sp<AMessage> response;
475    return PostAndAwaitResponse(msg, &response);
476}
477
478status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const {
479    sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, id());
480
481    sp<AMessage> response;
482    status_t err;
483    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
484        return err;
485    }
486
487    CHECK(response->findMessage("format", format));
488
489    return OK;
490}
491
492status_t MediaCodec::getInputFormat(sp<AMessage> *format) const {
493    sp<AMessage> msg = new AMessage(kWhatGetInputFormat, id());
494
495    sp<AMessage> response;
496    status_t err;
497    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
498        return err;
499    }
500
501    CHECK(response->findMessage("format", format));
502
503    return OK;
504}
505
506status_t MediaCodec::getName(AString *name) const {
507    sp<AMessage> msg = new AMessage(kWhatGetName, id());
508
509    sp<AMessage> response;
510    status_t err;
511    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
512        return err;
513    }
514
515    CHECK(response->findString("name", name));
516
517    return OK;
518}
519
520status_t MediaCodec::getInputBuffers(Vector<sp<ABuffer> > *buffers) const {
521    sp<AMessage> msg = new AMessage(kWhatGetBuffers, id());
522    msg->setInt32("portIndex", kPortIndexInput);
523    msg->setPointer("buffers", buffers);
524
525    sp<AMessage> response;
526    return PostAndAwaitResponse(msg, &response);
527}
528
529status_t MediaCodec::getOutputBuffers(Vector<sp<ABuffer> > *buffers) const {
530    sp<AMessage> msg = new AMessage(kWhatGetBuffers, id());
531    msg->setInt32("portIndex", kPortIndexOutput);
532    msg->setPointer("buffers", buffers);
533
534    sp<AMessage> response;
535    return PostAndAwaitResponse(msg, &response);
536}
537
538status_t MediaCodec::getOutputBuffer(size_t index, sp<ABuffer> *buffer) {
539    sp<AMessage> format;
540    return getBufferAndFormat(kPortIndexOutput, index, buffer, &format);
541}
542
543status_t MediaCodec::getOutputFormat(size_t index, sp<AMessage> *format) {
544    sp<ABuffer> buffer;
545    return getBufferAndFormat(kPortIndexOutput, index, &buffer, format);
546}
547
548status_t MediaCodec::getInputBuffer(size_t index, sp<ABuffer> *buffer) {
549    sp<AMessage> format;
550    return getBufferAndFormat(kPortIndexInput, index, buffer, &format);
551}
552
553bool MediaCodec::isExecuting() const {
554    return mState == STARTED || mState == FLUSHED;
555}
556
557status_t MediaCodec::getBufferAndFormat(
558        size_t portIndex, size_t index,
559        sp<ABuffer> *buffer, sp<AMessage> *format) {
560    // use mutex instead of a context switch
561
562    buffer->clear();
563    format->clear();
564    if (!isExecuting()) {
565        return INVALID_OPERATION;
566    }
567
568    // we do not want mPortBuffers to change during this section
569    // we also don't want mOwnedByClient to change during this
570    Mutex::Autolock al(mBufferLock);
571    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
572    if (index < buffers->size()) {
573        const BufferInfo &info = buffers->itemAt(index);
574        if (info.mOwnedByClient) {
575            *buffer = info.mData;
576            *format = info.mFormat;
577        }
578    }
579    return OK;
580}
581
582status_t MediaCodec::flush() {
583    sp<AMessage> msg = new AMessage(kWhatFlush, id());
584
585    sp<AMessage> response;
586    return PostAndAwaitResponse(msg, &response);
587}
588
589status_t MediaCodec::requestIDRFrame() {
590    (new AMessage(kWhatRequestIDRFrame, id()))->post();
591
592    return OK;
593}
594
595void MediaCodec::requestActivityNotification(const sp<AMessage> &notify) {
596    sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, id());
597    msg->setMessage("notify", notify);
598    msg->post();
599}
600
601////////////////////////////////////////////////////////////////////////////////
602
603void MediaCodec::cancelPendingDequeueOperations() {
604    if (mFlags & kFlagDequeueInputPending) {
605        PostReplyWithError(mDequeueInputReplyID, INVALID_OPERATION);
606
607        ++mDequeueInputTimeoutGeneration;
608        mDequeueInputReplyID = 0;
609        mFlags &= ~kFlagDequeueInputPending;
610    }
611
612    if (mFlags & kFlagDequeueOutputPending) {
613        PostReplyWithError(mDequeueOutputReplyID, INVALID_OPERATION);
614
615        ++mDequeueOutputTimeoutGeneration;
616        mDequeueOutputReplyID = 0;
617        mFlags &= ~kFlagDequeueOutputPending;
618    }
619}
620
621bool MediaCodec::handleDequeueInputBuffer(uint32_t replyID, bool newRequest) {
622    if (!isExecuting() || (mFlags & kFlagIsAsync)
623            || (mFlags & kFlagStickyError)
624            || (newRequest && (mFlags & kFlagDequeueInputPending))) {
625        PostReplyWithError(replyID, INVALID_OPERATION);
626        return true;
627    }
628
629    ssize_t index = dequeuePortBuffer(kPortIndexInput);
630
631    if (index < 0) {
632        CHECK_EQ(index, -EAGAIN);
633        return false;
634    }
635
636    sp<AMessage> response = new AMessage;
637    response->setSize("index", index);
638    response->postReply(replyID);
639
640    return true;
641}
642
643bool MediaCodec::handleDequeueOutputBuffer(uint32_t replyID, bool newRequest) {
644    sp<AMessage> response = new AMessage;
645
646    if (!isExecuting() || (mFlags & kFlagIsAsync)
647            || (mFlags & kFlagStickyError)
648            || (newRequest && (mFlags & kFlagDequeueOutputPending))) {
649        response->setInt32("err", INVALID_OPERATION);
650    } else if (mFlags & kFlagOutputBuffersChanged) {
651        response->setInt32("err", INFO_OUTPUT_BUFFERS_CHANGED);
652        mFlags &= ~kFlagOutputBuffersChanged;
653    } else if (mFlags & kFlagOutputFormatChanged) {
654        response->setInt32("err", INFO_FORMAT_CHANGED);
655        mFlags &= ~kFlagOutputFormatChanged;
656    } else {
657        ssize_t index = dequeuePortBuffer(kPortIndexOutput);
658
659        if (index < 0) {
660            CHECK_EQ(index, -EAGAIN);
661            return false;
662        }
663
664        const sp<ABuffer> &buffer =
665            mPortBuffers[kPortIndexOutput].itemAt(index).mData;
666
667        response->setSize("index", index);
668        response->setSize("offset", buffer->offset());
669        response->setSize("size", buffer->size());
670
671        int64_t timeUs;
672        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
673
674        response->setInt64("timeUs", timeUs);
675
676        int32_t omxFlags;
677        CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags));
678
679        uint32_t flags = 0;
680        if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) {
681            flags |= BUFFER_FLAG_SYNCFRAME;
682        }
683        if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) {
684            flags |= BUFFER_FLAG_CODECCONFIG;
685        }
686        if (omxFlags & OMX_BUFFERFLAG_EOS) {
687            flags |= BUFFER_FLAG_EOS;
688        }
689
690        response->setInt32("flags", flags);
691    }
692
693    response->postReply(replyID);
694
695    return true;
696}
697
698void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
699    switch (msg->what()) {
700        case kWhatCodecNotify:
701        {
702            int32_t what;
703            CHECK(msg->findInt32("what", &what));
704
705            switch (what) {
706                case CodecBase::kWhatError:
707                {
708                    int32_t omxError, internalError;
709                    CHECK(msg->findInt32("omx-error", &omxError));
710                    CHECK(msg->findInt32("err", &internalError));
711
712                    ALOGE("Codec reported an error. "
713                          "(omx error 0x%08x, internalError %d)",
714                          omxError, internalError);
715
716                    if (omxError == OMX_ErrorResourcesLost
717                            && internalError == DEAD_OBJECT) {
718                        mFlags |= kFlagSawMediaServerDie;
719                    }
720
721                    bool sendErrorReponse = true;
722
723                    switch (mState) {
724                        case INITIALIZING:
725                        {
726                            setState(UNINITIALIZED);
727                            break;
728                        }
729
730                        case CONFIGURING:
731                        {
732                            setState(INITIALIZED);
733                            break;
734                        }
735
736                        case STARTING:
737                        {
738                            setState(CONFIGURED);
739                            break;
740                        }
741
742                        case STOPPING:
743                        case RELEASING:
744                        {
745                            // Ignore the error, assuming we'll still get
746                            // the shutdown complete notification.
747
748                            sendErrorReponse = false;
749
750                            if (mFlags & kFlagSawMediaServerDie) {
751                                // MediaServer died, there definitely won't
752                                // be a shutdown complete notification after
753                                // all.
754
755                                // note that we're directly going from
756                                // STOPPING->UNINITIALIZED, instead of the
757                                // usual STOPPING->INITIALIZED state.
758                                setState(UNINITIALIZED);
759
760                                (new AMessage)->postReply(mReplyID);
761                            }
762                            break;
763                        }
764
765                        case FLUSHING:
766                        {
767                            setState(
768                                    (mFlags & kFlagIsAsync) ? FLUSHED : STARTED);
769                            break;
770                        }
771
772                        case FLUSHED:
773                        case STARTED:
774                        {
775                            sendErrorReponse = false;
776
777                            mFlags |= kFlagStickyError;
778                            postActivityNotificationIfPossible();
779
780                            cancelPendingDequeueOperations();
781
782                            if (mFlags & kFlagIsAsync) {
783                                onError(omxError, 0);
784                            }
785                            setState(UNINITIALIZED);
786                            break;
787                        }
788
789                        default:
790                        {
791                            sendErrorReponse = false;
792
793                            mFlags |= kFlagStickyError;
794                            postActivityNotificationIfPossible();
795
796                            if (mFlags & kFlagIsAsync) {
797                                onError(omxError, 0);
798                            }
799                            setState(UNINITIALIZED);
800                            break;
801                        }
802                    }
803
804                    if (sendErrorReponse) {
805                        PostReplyWithError(mReplyID, UNKNOWN_ERROR);
806                    }
807                    break;
808                }
809
810                case CodecBase::kWhatComponentAllocated:
811                {
812                    CHECK_EQ(mState, INITIALIZING);
813                    setState(INITIALIZED);
814
815                    CHECK(msg->findString("componentName", &mComponentName));
816
817                    if (mComponentName.startsWith("OMX.google.")) {
818                        mFlags |= kFlagIsSoftwareCodec;
819                    } else {
820                        mFlags &= ~kFlagIsSoftwareCodec;
821                    }
822
823                    if (mComponentName.endsWith(".secure")) {
824                        mFlags |= kFlagIsSecure;
825                    } else {
826                        mFlags &= ~kFlagIsSecure;
827                    }
828
829                    (new AMessage)->postReply(mReplyID);
830                    break;
831                }
832
833                case CodecBase::kWhatComponentConfigured:
834                {
835                    CHECK_EQ(mState, CONFIGURING);
836
837                    // reset input surface flag
838                    mHaveInputSurface = false;
839
840                    CHECK(msg->findMessage("input-format", &mInputFormat));
841                    CHECK(msg->findMessage("output-format", &mOutputFormat));
842
843                    setState(CONFIGURED);
844                    (new AMessage)->postReply(mReplyID);
845                    break;
846                }
847
848                case CodecBase::kWhatInputSurfaceCreated:
849                {
850                    // response to initiateCreateInputSurface()
851                    status_t err = NO_ERROR;
852                    sp<AMessage> response = new AMessage();
853                    if (!msg->findInt32("err", &err)) {
854                        sp<RefBase> obj;
855                        msg->findObject("input-surface", &obj);
856                        CHECK(obj != NULL);
857                        response->setObject("input-surface", obj);
858                        mHaveInputSurface = true;
859                    } else {
860                        response->setInt32("err", err);
861                    }
862                    response->postReply(mReplyID);
863                    break;
864                }
865
866                case CodecBase::kWhatSignaledInputEOS:
867                {
868                    // response to signalEndOfInputStream()
869                    sp<AMessage> response = new AMessage();
870                    status_t err;
871                    if (msg->findInt32("err", &err)) {
872                        response->setInt32("err", err);
873                    }
874                    response->postReply(mReplyID);
875                    break;
876                }
877
878
879                case CodecBase::kWhatBuffersAllocated:
880                {
881                    Mutex::Autolock al(mBufferLock);
882                    int32_t portIndex;
883                    CHECK(msg->findInt32("portIndex", &portIndex));
884
885                    ALOGV("%s buffers allocated",
886                          portIndex == kPortIndexInput ? "input" : "output");
887
888                    CHECK(portIndex == kPortIndexInput
889                            || portIndex == kPortIndexOutput);
890
891                    mPortBuffers[portIndex].clear();
892
893                    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
894
895                    sp<RefBase> obj;
896                    CHECK(msg->findObject("portDesc", &obj));
897
898                    sp<CodecBase::PortDescription> portDesc =
899                        static_cast<CodecBase::PortDescription *>(obj.get());
900
901                    size_t numBuffers = portDesc->countBuffers();
902
903                    for (size_t i = 0; i < numBuffers; ++i) {
904                        BufferInfo info;
905                        info.mBufferID = portDesc->bufferIDAt(i);
906                        info.mOwnedByClient = false;
907                        info.mData = portDesc->bufferAt(i);
908
909                        if (portIndex == kPortIndexInput && mCrypto != NULL) {
910                            info.mEncryptedData =
911                                new ABuffer(info.mData->capacity());
912                        }
913
914                        buffers->push_back(info);
915                    }
916
917                    if (portIndex == kPortIndexOutput) {
918                        if (mState == STARTING) {
919                            // We're always allocating output buffers after
920                            // allocating input buffers, so this is a good
921                            // indication that now all buffers are allocated.
922                            setState(STARTED);
923                            (new AMessage)->postReply(mReplyID);
924                        } else {
925                            mFlags |= kFlagOutputBuffersChanged;
926                            postActivityNotificationIfPossible();
927                        }
928                    }
929                    break;
930                }
931
932                case CodecBase::kWhatOutputFormatChanged:
933                {
934                    ALOGV("codec output format changed");
935
936                    if ((mFlags & kFlagIsSoftwareCodec)
937                            && mNativeWindow != NULL) {
938                        AString mime;
939                        CHECK(msg->findString("mime", &mime));
940
941                        if (!strncasecmp("video/", mime.c_str(), 6)) {
942                            delete mSoftRenderer;
943                            mSoftRenderer = NULL;
944
945                            int32_t width, height;
946                            CHECK(msg->findInt32("width", &width));
947                            CHECK(msg->findInt32("height", &height));
948
949                            int32_t cropLeft, cropTop, cropRight, cropBottom;
950                            CHECK(msg->findRect("crop",
951                                &cropLeft, &cropTop, &cropRight, &cropBottom));
952
953                            int32_t colorFormat;
954                            CHECK(msg->findInt32(
955                                        "color-format", &colorFormat));
956
957                            sp<MetaData> meta = new MetaData;
958                            meta->setInt32(kKeyWidth, width);
959                            meta->setInt32(kKeyHeight, height);
960                            meta->setRect(kKeyCropRect,
961                                cropLeft, cropTop, cropRight, cropBottom);
962                            meta->setInt32(kKeyColorFormat, colorFormat);
963
964                            mSoftRenderer =
965                                new SoftwareRenderer(mNativeWindow, meta);
966                        }
967                    }
968
969                    mOutputFormat = msg;
970
971                    if (mFlags & kFlagIsEncoder) {
972                        // Before we announce the format change we should
973                        // collect codec specific data and amend the output
974                        // format as necessary.
975                        mFlags |= kFlagGatherCodecSpecificData;
976                    } else if (mFlags & kFlagIsAsync) {
977                        onOutputFormatChanged();
978                    } else {
979                        mFlags |= kFlagOutputFormatChanged;
980                        postActivityNotificationIfPossible();
981                    }
982                    break;
983                }
984
985                case CodecBase::kWhatFillThisBuffer:
986                {
987                    /* size_t index = */updateBuffers(kPortIndexInput, msg);
988
989                    if (mState == FLUSHING
990                            || mState == STOPPING
991                            || mState == RELEASING) {
992                        returnBuffersToCodecOnPort(kPortIndexInput);
993                        break;
994                    }
995
996                    if (!mCSD.empty()) {
997                        ssize_t index = dequeuePortBuffer(kPortIndexInput);
998                        CHECK_GE(index, 0);
999
1000                        // If codec specific data had been specified as
1001                        // part of the format in the call to configure and
1002                        // if there's more csd left, we submit it here
1003                        // clients only get access to input buffers once
1004                        // this data has been exhausted.
1005
1006                        status_t err = queueCSDInputBuffer(index);
1007
1008                        if (err != OK) {
1009                            ALOGE("queueCSDInputBuffer failed w/ error %d",
1010                                  err);
1011
1012                            mFlags |= kFlagStickyError;
1013                            postActivityNotificationIfPossible();
1014
1015                            cancelPendingDequeueOperations();
1016                        }
1017                        break;
1018                    }
1019
1020                    if (mFlags & kFlagIsAsync) {
1021                        onInputBufferAvailable();
1022                    } else if (mFlags & kFlagDequeueInputPending) {
1023                        CHECK(handleDequeueInputBuffer(mDequeueInputReplyID));
1024
1025                        ++mDequeueInputTimeoutGeneration;
1026                        mFlags &= ~kFlagDequeueInputPending;
1027                        mDequeueInputReplyID = 0;
1028                    } else {
1029                        postActivityNotificationIfPossible();
1030                    }
1031                    break;
1032                }
1033
1034                case CodecBase::kWhatDrainThisBuffer:
1035                {
1036                    /* size_t index = */updateBuffers(kPortIndexOutput, msg);
1037
1038                    if (mState == FLUSHING
1039                            || mState == STOPPING
1040                            || mState == RELEASING) {
1041                        returnBuffersToCodecOnPort(kPortIndexOutput);
1042                        break;
1043                    }
1044
1045                    sp<ABuffer> buffer;
1046                    CHECK(msg->findBuffer("buffer", &buffer));
1047
1048                    int32_t omxFlags;
1049                    CHECK(msg->findInt32("flags", &omxFlags));
1050
1051                    buffer->meta()->setInt32("omxFlags", omxFlags);
1052
1053                    if (mFlags & kFlagGatherCodecSpecificData) {
1054                        // This is the very first output buffer after a
1055                        // format change was signalled, it'll either contain
1056                        // the one piece of codec specific data we can expect
1057                        // or there won't be codec specific data.
1058                        if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) {
1059                            status_t err =
1060                                amendOutputFormatWithCodecSpecificData(buffer);
1061
1062                            if (err != OK) {
1063                                ALOGE("Codec spit out malformed codec "
1064                                      "specific data!");
1065                            }
1066                        }
1067
1068                        mFlags &= ~kFlagGatherCodecSpecificData;
1069                        if (mFlags & kFlagIsAsync) {
1070                            onOutputFormatChanged();
1071                        } else {
1072                            mFlags |= kFlagOutputFormatChanged;
1073                        }
1074                    }
1075
1076                    if (mFlags & kFlagIsAsync) {
1077                        onOutputBufferAvailable();
1078                    } else if (mFlags & kFlagDequeueOutputPending) {
1079                        CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID));
1080
1081                        ++mDequeueOutputTimeoutGeneration;
1082                        mFlags &= ~kFlagDequeueOutputPending;
1083                        mDequeueOutputReplyID = 0;
1084                    } else {
1085                        postActivityNotificationIfPossible();
1086                    }
1087
1088                    break;
1089                }
1090
1091                case CodecBase::kWhatEOS:
1092                {
1093                    // We already notify the client of this by using the
1094                    // corresponding flag in "onOutputBufferReady".
1095                    break;
1096                }
1097
1098                case CodecBase::kWhatShutdownCompleted:
1099                {
1100                    if (mState == STOPPING) {
1101                        setState(INITIALIZED);
1102                    } else {
1103                        CHECK_EQ(mState, RELEASING);
1104                        setState(UNINITIALIZED);
1105                    }
1106
1107                    (new AMessage)->postReply(mReplyID);
1108                    break;
1109                }
1110
1111                case CodecBase::kWhatFlushCompleted:
1112                {
1113                    CHECK_EQ(mState, FLUSHING);
1114
1115                    if (mFlags & kFlagIsAsync) {
1116                        setState(FLUSHED);
1117                    } else {
1118                        setState(STARTED);
1119                        mCodec->signalResume();
1120                    }
1121
1122                    (new AMessage)->postReply(mReplyID);
1123                    break;
1124                }
1125
1126                default:
1127                    TRESPASS();
1128            }
1129            break;
1130        }
1131
1132        case kWhatInit:
1133        {
1134            uint32_t replyID;
1135            CHECK(msg->senderAwaitsResponse(&replyID));
1136
1137            if (mState != UNINITIALIZED) {
1138                PostReplyWithError(replyID, INVALID_OPERATION);
1139                break;
1140            }
1141
1142            mReplyID = replyID;
1143            setState(INITIALIZING);
1144
1145            AString name;
1146            CHECK(msg->findString("name", &name));
1147
1148            int32_t nameIsType;
1149            int32_t encoder = false;
1150            CHECK(msg->findInt32("nameIsType", &nameIsType));
1151            if (nameIsType) {
1152                CHECK(msg->findInt32("encoder", &encoder));
1153            }
1154
1155            sp<AMessage> format = new AMessage;
1156
1157            if (nameIsType) {
1158                format->setString("mime", name.c_str());
1159                format->setInt32("encoder", encoder);
1160            } else {
1161                format->setString("componentName", name.c_str());
1162            }
1163
1164            mCodec->initiateAllocateComponent(format);
1165            break;
1166        }
1167
1168        case kWhatSetCallback:
1169        {
1170            uint32_t replyID;
1171            CHECK(msg->senderAwaitsResponse(&replyID));
1172
1173            if (mState == UNINITIALIZED
1174                    || mState == INITIALIZING
1175                    || isExecuting()) {
1176                // callback can't be set after codec is executing,
1177                // or before it's initialized (as the callback
1178                // will be cleared when it goes to INITIALIZED)
1179                PostReplyWithError(replyID, INVALID_OPERATION);
1180                break;
1181            }
1182
1183            sp<AMessage> callback;
1184            CHECK(msg->findMessage("callback", &callback));
1185
1186            mCallback = callback;
1187
1188            if (mCallback != NULL) {
1189                ALOGI("MediaCodec will operate in async mode");
1190                mFlags |= kFlagIsAsync;
1191            } else {
1192                mFlags &= ~kFlagIsAsync;
1193            }
1194
1195            sp<AMessage> response = new AMessage;
1196            response->postReply(replyID);
1197            break;
1198        }
1199
1200        case kWhatConfigure:
1201        {
1202            uint32_t replyID;
1203            CHECK(msg->senderAwaitsResponse(&replyID));
1204
1205            if (mState != INITIALIZED) {
1206                PostReplyWithError(replyID, INVALID_OPERATION);
1207                break;
1208            }
1209
1210            sp<RefBase> obj;
1211            if (!msg->findObject("native-window", &obj)) {
1212                obj.clear();
1213            }
1214
1215            sp<AMessage> format;
1216            CHECK(msg->findMessage("format", &format));
1217
1218            if (obj != NULL) {
1219                format->setObject("native-window", obj);
1220
1221                status_t err = setNativeWindow(
1222                    static_cast<NativeWindowWrapper *>(obj.get())
1223                        ->getSurfaceTextureClient());
1224
1225                if (err != OK) {
1226                    PostReplyWithError(replyID, err);
1227                    break;
1228                }
1229            } else {
1230                setNativeWindow(NULL);
1231            }
1232
1233            mReplyID = replyID;
1234            setState(CONFIGURING);
1235
1236            void *crypto;
1237            if (!msg->findPointer("crypto", &crypto)) {
1238                crypto = NULL;
1239            }
1240
1241            mCrypto = static_cast<ICrypto *>(crypto);
1242
1243            uint32_t flags;
1244            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1245
1246            if (flags & CONFIGURE_FLAG_ENCODE) {
1247                format->setInt32("encoder", true);
1248                mFlags |= kFlagIsEncoder;
1249            }
1250
1251            extractCSD(format);
1252
1253            mCodec->initiateConfigureComponent(format);
1254            break;
1255        }
1256
1257        case kWhatCreateInputSurface:
1258        {
1259            uint32_t replyID;
1260            CHECK(msg->senderAwaitsResponse(&replyID));
1261
1262            // Must be configured, but can't have been started yet.
1263            if (mState != CONFIGURED) {
1264                PostReplyWithError(replyID, INVALID_OPERATION);
1265                break;
1266            }
1267
1268            mReplyID = replyID;
1269            mCodec->initiateCreateInputSurface();
1270            break;
1271        }
1272
1273        case kWhatStart:
1274        {
1275            uint32_t replyID;
1276            CHECK(msg->senderAwaitsResponse(&replyID));
1277
1278            if (mState == FLUSHED) {
1279                mCodec->signalResume();
1280                PostReplyWithError(replyID, OK);
1281            } else if (mState != CONFIGURED) {
1282                PostReplyWithError(replyID, INVALID_OPERATION);
1283                break;
1284            }
1285
1286            mReplyID = replyID;
1287            setState(STARTING);
1288
1289            mCodec->initiateStart();
1290            break;
1291        }
1292
1293        case kWhatStop:
1294        case kWhatRelease:
1295        {
1296            State targetState =
1297                (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED;
1298
1299            uint32_t replyID;
1300            CHECK(msg->senderAwaitsResponse(&replyID));
1301
1302            if (mState != INITIALIZED
1303                    && mState != CONFIGURED && !isExecuting()) {
1304                // We may be in "UNINITIALIZED" state already without the
1305                // client being aware of this if media server died while
1306                // we were being stopped. The client would assume that
1307                // after stop() returned, it would be safe to call release()
1308                // and it should be in this case, no harm to allow a release()
1309                // if we're already uninitialized.
1310                // Similarly stopping a stopped MediaCodec should be benign.
1311                sp<AMessage> response = new AMessage;
1312                response->setInt32(
1313                        "err",
1314                        mState == targetState ? OK : INVALID_OPERATION);
1315
1316                response->postReply(replyID);
1317                break;
1318            }
1319
1320            if (mFlags & kFlagSawMediaServerDie) {
1321                // It's dead, Jim. Don't expect initiateShutdown to yield
1322                // any useful results now...
1323                setState(UNINITIALIZED);
1324                (new AMessage)->postReply(replyID);
1325                break;
1326            }
1327
1328            mReplyID = replyID;
1329            setState(msg->what() == kWhatStop ? STOPPING : RELEASING);
1330
1331            mCodec->initiateShutdown(
1332                    msg->what() == kWhatStop /* keepComponentAllocated */);
1333
1334            returnBuffersToCodec();
1335            break;
1336        }
1337
1338        case kWhatDequeueInputBuffer:
1339        {
1340            uint32_t replyID;
1341            CHECK(msg->senderAwaitsResponse(&replyID));
1342
1343            if (mFlags & kFlagIsAsync) {
1344                ALOGE("dequeueOutputBuffer can't be used in async mode");
1345                PostReplyWithError(replyID, INVALID_OPERATION);
1346                break;
1347            }
1348
1349            if (mHaveInputSurface) {
1350                ALOGE("dequeueInputBuffer can't be used with input surface");
1351                PostReplyWithError(replyID, INVALID_OPERATION);
1352                break;
1353            }
1354
1355            if (handleDequeueInputBuffer(replyID, true /* new request */)) {
1356                break;
1357            }
1358
1359            int64_t timeoutUs;
1360            CHECK(msg->findInt64("timeoutUs", &timeoutUs));
1361
1362            if (timeoutUs == 0ll) {
1363                PostReplyWithError(replyID, -EAGAIN);
1364                break;
1365            }
1366
1367            mFlags |= kFlagDequeueInputPending;
1368            mDequeueInputReplyID = replyID;
1369
1370            if (timeoutUs > 0ll) {
1371                sp<AMessage> timeoutMsg =
1372                    new AMessage(kWhatDequeueInputTimedOut, id());
1373                timeoutMsg->setInt32(
1374                        "generation", ++mDequeueInputTimeoutGeneration);
1375                timeoutMsg->post(timeoutUs);
1376            }
1377            break;
1378        }
1379
1380        case kWhatDequeueInputTimedOut:
1381        {
1382            int32_t generation;
1383            CHECK(msg->findInt32("generation", &generation));
1384
1385            if (generation != mDequeueInputTimeoutGeneration) {
1386                // Obsolete
1387                break;
1388            }
1389
1390            CHECK(mFlags & kFlagDequeueInputPending);
1391
1392            PostReplyWithError(mDequeueInputReplyID, -EAGAIN);
1393
1394            mFlags &= ~kFlagDequeueInputPending;
1395            mDequeueInputReplyID = 0;
1396            break;
1397        }
1398
1399        case kWhatQueueInputBuffer:
1400        {
1401            uint32_t replyID;
1402            CHECK(msg->senderAwaitsResponse(&replyID));
1403
1404            if (!isExecuting() || (mFlags & kFlagStickyError)) {
1405                PostReplyWithError(replyID, INVALID_OPERATION);
1406                break;
1407            }
1408
1409            status_t err = onQueueInputBuffer(msg);
1410
1411            PostReplyWithError(replyID, err);
1412            break;
1413        }
1414
1415        case kWhatDequeueOutputBuffer:
1416        {
1417            uint32_t replyID;
1418            CHECK(msg->senderAwaitsResponse(&replyID));
1419
1420            if (mFlags & kFlagIsAsync) {
1421                ALOGE("dequeueOutputBuffer can't be used in async mode");
1422                PostReplyWithError(replyID, INVALID_OPERATION);
1423                break;
1424            }
1425
1426            if (handleDequeueOutputBuffer(replyID, true /* new request */)) {
1427                break;
1428            }
1429
1430            int64_t timeoutUs;
1431            CHECK(msg->findInt64("timeoutUs", &timeoutUs));
1432
1433            if (timeoutUs == 0ll) {
1434                PostReplyWithError(replyID, -EAGAIN);
1435                break;
1436            }
1437
1438            mFlags |= kFlagDequeueOutputPending;
1439            mDequeueOutputReplyID = replyID;
1440
1441            if (timeoutUs > 0ll) {
1442                sp<AMessage> timeoutMsg =
1443                    new AMessage(kWhatDequeueOutputTimedOut, id());
1444                timeoutMsg->setInt32(
1445                        "generation", ++mDequeueOutputTimeoutGeneration);
1446                timeoutMsg->post(timeoutUs);
1447            }
1448            break;
1449        }
1450
1451        case kWhatDequeueOutputTimedOut:
1452        {
1453            int32_t generation;
1454            CHECK(msg->findInt32("generation", &generation));
1455
1456            if (generation != mDequeueOutputTimeoutGeneration) {
1457                // Obsolete
1458                break;
1459            }
1460
1461            CHECK(mFlags & kFlagDequeueOutputPending);
1462
1463            PostReplyWithError(mDequeueOutputReplyID, -EAGAIN);
1464
1465            mFlags &= ~kFlagDequeueOutputPending;
1466            mDequeueOutputReplyID = 0;
1467            break;
1468        }
1469
1470        case kWhatReleaseOutputBuffer:
1471        {
1472            uint32_t replyID;
1473            CHECK(msg->senderAwaitsResponse(&replyID));
1474
1475            if (!isExecuting() || (mFlags & kFlagStickyError)) {
1476                PostReplyWithError(replyID, INVALID_OPERATION);
1477                break;
1478            }
1479
1480            status_t err = onReleaseOutputBuffer(msg);
1481
1482            PostReplyWithError(replyID, err);
1483            break;
1484        }
1485
1486        case kWhatSignalEndOfInputStream:
1487        {
1488            uint32_t replyID;
1489            CHECK(msg->senderAwaitsResponse(&replyID));
1490
1491            if (!isExecuting() || (mFlags & kFlagStickyError)) {
1492                PostReplyWithError(replyID, INVALID_OPERATION);
1493                break;
1494            }
1495
1496            mReplyID = replyID;
1497            mCodec->signalEndOfInputStream();
1498            break;
1499        }
1500
1501        case kWhatGetBuffers:
1502        {
1503            uint32_t replyID;
1504            CHECK(msg->senderAwaitsResponse(&replyID));
1505
1506            if (!isExecuting() || (mFlags & kFlagIsAsync)
1507                    || (mFlags & kFlagStickyError)) {
1508                PostReplyWithError(replyID, INVALID_OPERATION);
1509                break;
1510            }
1511
1512            int32_t portIndex;
1513            CHECK(msg->findInt32("portIndex", &portIndex));
1514
1515            Vector<sp<ABuffer> > *dstBuffers;
1516            CHECK(msg->findPointer("buffers", (void **)&dstBuffers));
1517
1518            dstBuffers->clear();
1519            const Vector<BufferInfo> &srcBuffers = mPortBuffers[portIndex];
1520
1521            for (size_t i = 0; i < srcBuffers.size(); ++i) {
1522                const BufferInfo &info = srcBuffers.itemAt(i);
1523
1524                dstBuffers->push_back(
1525                        (portIndex == kPortIndexInput && mCrypto != NULL)
1526                                ? info.mEncryptedData : info.mData);
1527            }
1528
1529            (new AMessage)->postReply(replyID);
1530            break;
1531        }
1532
1533        case kWhatFlush:
1534        {
1535            uint32_t replyID;
1536            CHECK(msg->senderAwaitsResponse(&replyID));
1537
1538            if (!isExecuting() || (mFlags & kFlagStickyError)) {
1539                PostReplyWithError(replyID, INVALID_OPERATION);
1540                break;
1541            }
1542
1543            mReplyID = replyID;
1544            // TODO: skip flushing if already FLUSHED
1545            setState(FLUSHING);
1546
1547            mCodec->signalFlush();
1548            returnBuffersToCodec();
1549            break;
1550        }
1551
1552        case kWhatGetInputFormat:
1553        case kWhatGetOutputFormat:
1554        {
1555            sp<AMessage> format =
1556                (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat);
1557
1558            uint32_t replyID;
1559            CHECK(msg->senderAwaitsResponse(&replyID));
1560
1561            if ((mState != CONFIGURED && mState != STARTING &&
1562                 mState != STARTED && mState != FLUSHING &&
1563                 mState != FLUSHED)
1564                    || (mFlags & kFlagStickyError)
1565                    || format == NULL) {
1566                PostReplyWithError(replyID, INVALID_OPERATION);
1567                break;
1568            }
1569
1570            sp<AMessage> response = new AMessage;
1571            response->setMessage("format", format);
1572            response->postReply(replyID);
1573            break;
1574        }
1575
1576        case kWhatRequestIDRFrame:
1577        {
1578            mCodec->signalRequestIDRFrame();
1579            break;
1580        }
1581
1582        case kWhatRequestActivityNotification:
1583        {
1584            CHECK(mActivityNotify == NULL);
1585            CHECK(msg->findMessage("notify", &mActivityNotify));
1586
1587            postActivityNotificationIfPossible();
1588            break;
1589        }
1590
1591        case kWhatGetName:
1592        {
1593            uint32_t replyID;
1594            CHECK(msg->senderAwaitsResponse(&replyID));
1595
1596            if (mComponentName.empty()) {
1597                PostReplyWithError(replyID, INVALID_OPERATION);
1598                break;
1599            }
1600
1601            sp<AMessage> response = new AMessage;
1602            response->setString("name", mComponentName.c_str());
1603            response->postReply(replyID);
1604            break;
1605        }
1606
1607        case kWhatSetParameters:
1608        {
1609            uint32_t replyID;
1610            CHECK(msg->senderAwaitsResponse(&replyID));
1611
1612            sp<AMessage> params;
1613            CHECK(msg->findMessage("params", &params));
1614
1615            status_t err = onSetParameters(params);
1616
1617            PostReplyWithError(replyID, err);
1618            break;
1619        }
1620
1621        default:
1622            TRESPASS();
1623    }
1624}
1625
1626void MediaCodec::extractCSD(const sp<AMessage> &format) {
1627    mCSD.clear();
1628
1629    size_t i = 0;
1630    for (;;) {
1631        sp<ABuffer> csd;
1632        if (!format->findBuffer(StringPrintf("csd-%u", i).c_str(), &csd)) {
1633            break;
1634        }
1635
1636        mCSD.push_back(csd);
1637        ++i;
1638    }
1639
1640    ALOGV("Found %zu pieces of codec specific data.", mCSD.size());
1641}
1642
1643status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) {
1644    CHECK(!mCSD.empty());
1645
1646    const BufferInfo *info =
1647        &mPortBuffers[kPortIndexInput].itemAt(bufferIndex);
1648
1649    sp<ABuffer> csd = *mCSD.begin();
1650    mCSD.erase(mCSD.begin());
1651
1652    const sp<ABuffer> &codecInputData =
1653        (mCrypto != NULL) ? info->mEncryptedData : info->mData;
1654
1655    if (csd->size() > codecInputData->capacity()) {
1656        return -EINVAL;
1657    }
1658
1659    memcpy(codecInputData->data(), csd->data(), csd->size());
1660
1661    AString errorDetailMsg;
1662
1663    sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, id());
1664    msg->setSize("index", bufferIndex);
1665    msg->setSize("offset", 0);
1666    msg->setSize("size", csd->size());
1667    msg->setInt64("timeUs", 0ll);
1668    msg->setInt32("flags", BUFFER_FLAG_CODECCONFIG);
1669    msg->setPointer("errorDetailMsg", &errorDetailMsg);
1670
1671    return onQueueInputBuffer(msg);
1672}
1673
1674void MediaCodec::setState(State newState) {
1675    if (newState == INITIALIZED || newState == UNINITIALIZED) {
1676        delete mSoftRenderer;
1677        mSoftRenderer = NULL;
1678
1679        mCrypto.clear();
1680        setNativeWindow(NULL);
1681
1682        mInputFormat.clear();
1683        mOutputFormat.clear();
1684        mFlags &= ~kFlagOutputFormatChanged;
1685        mFlags &= ~kFlagOutputBuffersChanged;
1686        mFlags &= ~kFlagStickyError;
1687        mFlags &= ~kFlagIsEncoder;
1688        mFlags &= ~kFlagGatherCodecSpecificData;
1689        mFlags &= ~kFlagIsAsync;
1690
1691        mActivityNotify.clear();
1692        mCallback.clear();
1693    }
1694
1695    if (newState == UNINITIALIZED) {
1696        // return any straggling buffers, e.g. if we got here on an error
1697        returnBuffersToCodec();
1698
1699        mComponentName.clear();
1700
1701        // The component is gone, mediaserver's probably back up already
1702        // but should definitely be back up should we try to instantiate
1703        // another component.. and the cycle continues.
1704        mFlags &= ~kFlagSawMediaServerDie;
1705    }
1706
1707    mState = newState;
1708
1709    cancelPendingDequeueOperations();
1710
1711    updateBatteryStat();
1712}
1713
1714void MediaCodec::returnBuffersToCodec() {
1715    returnBuffersToCodecOnPort(kPortIndexInput);
1716    returnBuffersToCodecOnPort(kPortIndexOutput);
1717}
1718
1719void MediaCodec::returnBuffersToCodecOnPort(int32_t portIndex) {
1720    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
1721    Mutex::Autolock al(mBufferLock);
1722
1723    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
1724
1725    for (size_t i = 0; i < buffers->size(); ++i) {
1726        BufferInfo *info = &buffers->editItemAt(i);
1727
1728        if (info->mNotify != NULL) {
1729            sp<AMessage> msg = info->mNotify;
1730            info->mNotify = NULL;
1731            info->mOwnedByClient = false;
1732
1733            if (portIndex == kPortIndexInput) {
1734                /* no error, just returning buffers */
1735                msg->setInt32("err", OK);
1736            }
1737            msg->post();
1738        }
1739    }
1740
1741    mAvailPortBuffers[portIndex].clear();
1742}
1743
1744size_t MediaCodec::updateBuffers(
1745        int32_t portIndex, const sp<AMessage> &msg) {
1746    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
1747
1748    uint32_t bufferID;
1749    CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID));
1750
1751    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
1752
1753    for (size_t i = 0; i < buffers->size(); ++i) {
1754        BufferInfo *info = &buffers->editItemAt(i);
1755
1756        if (info->mBufferID == bufferID) {
1757            CHECK(info->mNotify == NULL);
1758            CHECK(msg->findMessage("reply", &info->mNotify));
1759
1760            mAvailPortBuffers[portIndex].push_back(i);
1761
1762            return i;
1763        }
1764    }
1765
1766    TRESPASS();
1767
1768    return 0;
1769}
1770
1771status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) {
1772    size_t index;
1773    size_t offset;
1774    size_t size;
1775    int64_t timeUs;
1776    uint32_t flags;
1777    CHECK(msg->findSize("index", &index));
1778    CHECK(msg->findSize("offset", &offset));
1779    CHECK(msg->findInt64("timeUs", &timeUs));
1780    CHECK(msg->findInt32("flags", (int32_t *)&flags));
1781
1782    const CryptoPlugin::SubSample *subSamples;
1783    size_t numSubSamples;
1784    const uint8_t *key;
1785    const uint8_t *iv;
1786    CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted;
1787
1788    // We allow the simpler queueInputBuffer API to be used even in
1789    // secure mode, by fabricating a single unencrypted subSample.
1790    CryptoPlugin::SubSample ss;
1791
1792    if (msg->findSize("size", &size)) {
1793        if (mCrypto != NULL) {
1794            ss.mNumBytesOfClearData = size;
1795            ss.mNumBytesOfEncryptedData = 0;
1796
1797            subSamples = &ss;
1798            numSubSamples = 1;
1799            key = NULL;
1800            iv = NULL;
1801        }
1802    } else {
1803        if (mCrypto == NULL) {
1804            return -EINVAL;
1805        }
1806
1807        CHECK(msg->findPointer("subSamples", (void **)&subSamples));
1808        CHECK(msg->findSize("numSubSamples", &numSubSamples));
1809        CHECK(msg->findPointer("key", (void **)&key));
1810        CHECK(msg->findPointer("iv", (void **)&iv));
1811
1812        int32_t tmp;
1813        CHECK(msg->findInt32("mode", &tmp));
1814
1815        mode = (CryptoPlugin::Mode)tmp;
1816
1817        size = 0;
1818        for (size_t i = 0; i < numSubSamples; ++i) {
1819            size += subSamples[i].mNumBytesOfClearData;
1820            size += subSamples[i].mNumBytesOfEncryptedData;
1821        }
1822    }
1823
1824    if (index >= mPortBuffers[kPortIndexInput].size()) {
1825        return -ERANGE;
1826    }
1827
1828    BufferInfo *info = &mPortBuffers[kPortIndexInput].editItemAt(index);
1829
1830    if (info->mNotify == NULL || !info->mOwnedByClient) {
1831        return -EACCES;
1832    }
1833
1834    if (offset + size > info->mData->capacity()) {
1835        return -EINVAL;
1836    }
1837
1838    sp<AMessage> reply = info->mNotify;
1839    info->mData->setRange(offset, size);
1840    info->mData->meta()->setInt64("timeUs", timeUs);
1841
1842    if (flags & BUFFER_FLAG_EOS) {
1843        info->mData->meta()->setInt32("eos", true);
1844    }
1845
1846    if (flags & BUFFER_FLAG_CODECCONFIG) {
1847        info->mData->meta()->setInt32("csd", true);
1848    }
1849
1850    if (mCrypto != NULL) {
1851        if (size > info->mEncryptedData->capacity()) {
1852            return -ERANGE;
1853        }
1854
1855        AString *errorDetailMsg;
1856        CHECK(msg->findPointer("errorDetailMsg", (void **)&errorDetailMsg));
1857
1858        ssize_t result = mCrypto->decrypt(
1859                (mFlags & kFlagIsSecure) != 0,
1860                key,
1861                iv,
1862                mode,
1863                info->mEncryptedData->base() + offset,
1864                subSamples,
1865                numSubSamples,
1866                info->mData->base(),
1867                errorDetailMsg);
1868
1869        if (result < 0) {
1870            return result;
1871        }
1872
1873        info->mData->setRange(0, result);
1874    }
1875
1876    // synchronization boundary for getBufferAndFormat
1877    {
1878        Mutex::Autolock al(mBufferLock);
1879        info->mOwnedByClient = false;
1880    }
1881    reply->setBuffer("buffer", info->mData);
1882    reply->post();
1883
1884    info->mNotify = NULL;
1885
1886    return OK;
1887}
1888
1889status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
1890    size_t index;
1891    CHECK(msg->findSize("index", &index));
1892
1893    int32_t render;
1894    if (!msg->findInt32("render", &render)) {
1895        render = 0;
1896    }
1897
1898    if (!isExecuting()) {
1899        return -EINVAL;
1900    }
1901
1902    if (index >= mPortBuffers[kPortIndexOutput].size()) {
1903        return -ERANGE;
1904    }
1905
1906    BufferInfo *info = &mPortBuffers[kPortIndexOutput].editItemAt(index);
1907
1908    if (info->mNotify == NULL || !info->mOwnedByClient) {
1909        return -EACCES;
1910    }
1911
1912    // synchronization boundary for getBufferAndFormat
1913    {
1914        Mutex::Autolock al(mBufferLock);
1915        info->mOwnedByClient = false;
1916    }
1917
1918    if (render && info->mData != NULL && info->mData->size() != 0) {
1919        info->mNotify->setInt32("render", true);
1920
1921        int64_t timestampNs = 0;
1922        if (msg->findInt64("timestampNs", &timestampNs)) {
1923            info->mNotify->setInt64("timestampNs", timestampNs);
1924        } else {
1925            // TODO: it seems like we should use the timestamp
1926            // in the (media)buffer as it potentially came from
1927            // an input surface, but we did not propagate it prior to
1928            // API 20.  Perhaps check for target SDK version.
1929#if 0
1930            if (info->mData->meta()->findInt64("timeUs", &timestampNs)) {
1931                ALOGV("using buffer PTS of %" PRId64, timestampNs);
1932                timestampNs *= 1000;
1933            }
1934#endif
1935        }
1936
1937        if (mSoftRenderer != NULL) {
1938            mSoftRenderer->render(
1939                    info->mData->data(), info->mData->size(), timestampNs, NULL);
1940        }
1941    }
1942
1943    info->mNotify->post();
1944    info->mNotify = NULL;
1945
1946    return OK;
1947}
1948
1949ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {
1950    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
1951
1952    List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];
1953
1954    if (availBuffers->empty()) {
1955        return -EAGAIN;
1956    }
1957
1958    size_t index = *availBuffers->begin();
1959    availBuffers->erase(availBuffers->begin());
1960
1961    BufferInfo *info = &mPortBuffers[portIndex].editItemAt(index);
1962    CHECK(!info->mOwnedByClient);
1963    {
1964        Mutex::Autolock al(mBufferLock);
1965        info->mFormat = portIndex == kPortIndexInput ? mInputFormat : mOutputFormat;
1966        info->mOwnedByClient = true;
1967
1968        // set image-data
1969        if (info->mFormat != NULL) {
1970            sp<ABuffer> imageData;
1971            if (info->mFormat->findBuffer("image-data", &imageData)) {
1972                info->mData->meta()->setBuffer("image-data", imageData);
1973            }
1974            int32_t left, top, right, bottom;
1975            if (info->mFormat->findRect("crop", &left, &top, &right, &bottom)) {
1976                info->mData->meta()->setRect("crop-rect", left, top, right, bottom);
1977            }
1978        }
1979    }
1980
1981    return index;
1982}
1983
1984status_t MediaCodec::setNativeWindow(
1985        const sp<Surface> &surfaceTextureClient) {
1986    status_t err;
1987
1988    if (mNativeWindow != NULL) {
1989        err = native_window_api_disconnect(
1990                mNativeWindow.get(), NATIVE_WINDOW_API_MEDIA);
1991
1992        if (err != OK) {
1993            ALOGW("native_window_api_disconnect returned an error: %s (%d)",
1994                    strerror(-err), err);
1995        }
1996
1997        mNativeWindow.clear();
1998    }
1999
2000    if (surfaceTextureClient != NULL) {
2001        err = native_window_api_connect(
2002                surfaceTextureClient.get(), NATIVE_WINDOW_API_MEDIA);
2003
2004        if (err != OK) {
2005            ALOGE("native_window_api_connect returned an error: %s (%d)",
2006                    strerror(-err), err);
2007
2008            return err;
2009        }
2010
2011        mNativeWindow = surfaceTextureClient;
2012    }
2013
2014    return OK;
2015}
2016
2017void MediaCodec::onInputBufferAvailable() {
2018    int32_t index;
2019    while ((index = dequeuePortBuffer(kPortIndexInput)) >= 0) {
2020        sp<AMessage> msg = mCallback->dup();
2021        msg->setInt32("callbackID", CB_INPUT_AVAILABLE);
2022        msg->setInt32("index", index);
2023        msg->post();
2024    }
2025}
2026
2027void MediaCodec::onOutputBufferAvailable() {
2028    int32_t index;
2029    while ((index = dequeuePortBuffer(kPortIndexOutput)) >= 0) {
2030        const sp<ABuffer> &buffer =
2031            mPortBuffers[kPortIndexOutput].itemAt(index).mData;
2032        sp<AMessage> msg = mCallback->dup();
2033        msg->setInt32("callbackID", CB_OUTPUT_AVAILABLE);
2034        msg->setInt32("index", index);
2035        msg->setSize("offset", buffer->offset());
2036        msg->setSize("size", buffer->size());
2037
2038        int64_t timeUs;
2039        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2040
2041        msg->setInt64("timeUs", timeUs);
2042
2043        int32_t omxFlags;
2044        CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags));
2045
2046        uint32_t flags = 0;
2047        if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) {
2048            flags |= BUFFER_FLAG_SYNCFRAME;
2049        }
2050        if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) {
2051            flags |= BUFFER_FLAG_CODECCONFIG;
2052        }
2053        if (omxFlags & OMX_BUFFERFLAG_EOS) {
2054            flags |= BUFFER_FLAG_EOS;
2055        }
2056
2057        msg->setInt32("flags", flags);
2058
2059        msg->post();
2060    }
2061}
2062
2063void MediaCodec::onError(status_t err, int32_t actionCode, const char *detail) {
2064    if (mCallback != NULL) {
2065        sp<AMessage> msg = mCallback->dup();
2066        msg->setInt32("callbackID", CB_ERROR);
2067        msg->setInt32("err", err);
2068        msg->setInt32("actionCode", actionCode);
2069
2070        if (detail != NULL) {
2071            msg->setString("detail", detail);
2072        }
2073
2074        msg->post();
2075    }
2076}
2077
2078void MediaCodec::onOutputFormatChanged() {
2079    if (mCallback != NULL) {
2080        sp<AMessage> msg = mCallback->dup();
2081        msg->setInt32("callbackID", CB_OUTPUT_FORMAT_CHANGED);
2082        msg->setMessage("format", mOutputFormat);
2083        msg->post();
2084    }
2085}
2086
2087
2088void MediaCodec::postActivityNotificationIfPossible() {
2089    if (mActivityNotify == NULL) {
2090        return;
2091    }
2092
2093    if ((mFlags & (kFlagStickyError
2094                    | kFlagOutputBuffersChanged
2095                    | kFlagOutputFormatChanged))
2096            || !mAvailPortBuffers[kPortIndexInput].empty()
2097            || !mAvailPortBuffers[kPortIndexOutput].empty()) {
2098        mActivityNotify->post();
2099        mActivityNotify.clear();
2100    }
2101}
2102
2103status_t MediaCodec::setParameters(const sp<AMessage> &params) {
2104    sp<AMessage> msg = new AMessage(kWhatSetParameters, id());
2105    msg->setMessage("params", params);
2106
2107    sp<AMessage> response;
2108    return PostAndAwaitResponse(msg, &response);
2109}
2110
2111status_t MediaCodec::onSetParameters(const sp<AMessage> &params) {
2112    mCodec->signalSetParameters(params);
2113
2114    return OK;
2115}
2116
2117status_t MediaCodec::amendOutputFormatWithCodecSpecificData(
2118        const sp<ABuffer> &buffer) {
2119    AString mime;
2120    CHECK(mOutputFormat->findString("mime", &mime));
2121
2122    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) {
2123        // Codec specific data should be SPS and PPS in a single buffer,
2124        // each prefixed by a startcode (0x00 0x00 0x00 0x01).
2125        // We separate the two and put them into the output format
2126        // under the keys "csd-0" and "csd-1".
2127
2128        unsigned csdIndex = 0;
2129
2130        const uint8_t *data = buffer->data();
2131        size_t size = buffer->size();
2132
2133        const uint8_t *nalStart;
2134        size_t nalSize;
2135        while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) {
2136            sp<ABuffer> csd = new ABuffer(nalSize + 4);
2137            memcpy(csd->data(), "\x00\x00\x00\x01", 4);
2138            memcpy(csd->data() + 4, nalStart, nalSize);
2139
2140            mOutputFormat->setBuffer(
2141                    StringPrintf("csd-%u", csdIndex).c_str(), csd);
2142
2143            ++csdIndex;
2144        }
2145
2146        if (csdIndex != 2) {
2147            return ERROR_MALFORMED;
2148        }
2149    } else {
2150        // For everything else we just stash the codec specific data into
2151        // the output format as a single piece of csd under "csd-0".
2152        mOutputFormat->setBuffer("csd-0", buffer);
2153    }
2154
2155    return OK;
2156}
2157
2158void MediaCodec::updateBatteryStat() {
2159    if (mState == CONFIGURED && !mBatteryStatNotified) {
2160        AString mime;
2161        CHECK(mOutputFormat != NULL &&
2162                mOutputFormat->findString("mime", &mime));
2163
2164        mIsVideo = mime.startsWithIgnoreCase("video/");
2165
2166        BatteryNotifier& notifier(BatteryNotifier::getInstance());
2167
2168        if (mIsVideo) {
2169            notifier.noteStartVideo();
2170        } else {
2171            notifier.noteStartAudio();
2172        }
2173
2174        mBatteryStatNotified = true;
2175    } else if (mState == UNINITIALIZED && mBatteryStatNotified) {
2176        BatteryNotifier& notifier(BatteryNotifier::getInstance());
2177
2178        if (mIsVideo) {
2179            notifier.noteStopVideo();
2180        } else {
2181            notifier.noteStopAudio();
2182        }
2183
2184        mBatteryStatNotified = false;
2185    }
2186}
2187
2188}  // namespace android
2189