MediaCodec.cpp revision 58828196edf2fc4debbd7913198a8149f039b4a9 
1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "MediaCodec"
19#include <inttypes.h>
20
21#include "include/avc_utils.h"
22#include "include/SoftwareRenderer.h"
23
24#include <binder/IMemory.h>
25#include <binder/IPCThreadState.h>
26#include <binder/IServiceManager.h>
27#include <binder/MemoryDealer.h>
28#include <gui/BufferQueue.h>
29#include <gui/Surface.h>
30#include <media/ICrypto.h>
31#include <media/IOMX.h>
32#include <media/IResourceManagerService.h>
33#include <media/stagefright/foundation/ABuffer.h>
34#include <media/stagefright/foundation/ADebug.h>
35#include <media/stagefright/foundation/AMessage.h>
36#include <media/stagefright/foundation/AString.h>
37#include <media/stagefright/foundation/hexdump.h>
38#include <media/stagefright/ACodec.h>
39#include <media/stagefright/BufferProducerWrapper.h>
40#include <media/stagefright/MediaCodec.h>
41#include <media/stagefright/MediaCodecList.h>
42#include <media/stagefright/MediaDefs.h>
43#include <media/stagefright/MediaErrors.h>
44#include <media/stagefright/MediaFilter.h>
45#include <media/stagefright/MetaData.h>
46#include <media/stagefright/OMXClient.h>
47#include <media/stagefright/OMXCodec.h>
48#include <media/stagefright/PersistentSurface.h>
49#include <media/stagefright/SurfaceUtils.h>
50#include <mediautils/BatteryNotifier.h>
51#include <private/android_filesystem_config.h>
52#include <utils/Log.h>
53#include <utils/Singleton.h>
54
55namespace android {
56
57static int64_t getId(sp<IResourceManagerClient> client) {
58    return (int64_t) client.get();
59}
60
61static bool isResourceError(status_t err) {
62    return (err == NO_MEMORY);
63}
64
65static const int kMaxRetry = 2;
66
67struct ResourceManagerClient : public BnResourceManagerClient {
68    ResourceManagerClient(MediaCodec* codec) : mMediaCodec(codec) {}
69
70    virtual bool reclaimResource() {
71        sp<MediaCodec> codec = mMediaCodec.promote();
72        if (codec == NULL) {
73            // codec is already gone.
74            return true;
75        }
76        status_t err = codec->reclaim();
77        if (err != OK) {
78            ALOGW("ResourceManagerClient failed to release codec with err %d", err);
79        }
80        return (err == OK);
81    }
82
83    virtual String8 getName() {
84        String8 ret;
85        sp<MediaCodec> codec = mMediaCodec.promote();
86        if (codec == NULL) {
87            // codec is already gone.
88            return ret;
89        }
90
91        AString name;
92        if (codec->getName(&name) == OK) {
93            ret.setTo(name.c_str());
94        }
95        return ret;
96    }
97
98protected:
99    virtual ~ResourceManagerClient() {}
100
101private:
102    wp<MediaCodec> mMediaCodec;
103
104    DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient);
105};
106
107MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy(pid_t pid)
108        : mPid(pid) {
109    if (mPid == MediaCodec::kNoPid) {
110        mPid = IPCThreadState::self()->getCallingPid();
111    }
112}
113
114MediaCodec::ResourceManagerServiceProxy::~ResourceManagerServiceProxy() {
115    if (mService != NULL) {
116        IInterface::asBinder(mService)->unlinkToDeath(this);
117    }
118}
119
120void MediaCodec::ResourceManagerServiceProxy::init() {
121    sp<IServiceManager> sm = defaultServiceManager();
122    sp<IBinder> binder = sm->getService(String16("media.resource_manager"));
123    mService = interface_cast<IResourceManagerService>(binder);
124    if (mService == NULL) {
125        ALOGE("Failed to get ResourceManagerService");
126        return;
127    }
128    IInterface::asBinder(mService)->linkToDeath(this);
129}
130
131void MediaCodec::ResourceManagerServiceProxy::binderDied(const wp<IBinder>& /*who*/) {
132    ALOGW("ResourceManagerService died.");
133    Mutex::Autolock _l(mLock);
134    mService.clear();
135}
136
137void MediaCodec::ResourceManagerServiceProxy::addResource(
138        int64_t clientId,
139        const sp<IResourceManagerClient> client,
140        const Vector<MediaResource> &resources) {
141    Mutex::Autolock _l(mLock);
142    if (mService == NULL) {
143        return;
144    }
145    mService->addResource(mPid, clientId, client, resources);
146}
147
148void MediaCodec::ResourceManagerServiceProxy::removeResource(int64_t clientId) {
149    Mutex::Autolock _l(mLock);
150    if (mService == NULL) {
151        return;
152    }
153    mService->removeResource(mPid, clientId);
154}
155
156bool MediaCodec::ResourceManagerServiceProxy::reclaimResource(
157        const Vector<MediaResource> &resources) {
158    Mutex::Autolock _l(mLock);
159    if (mService == NULL) {
160        return false;
161    }
162    return mService->reclaimResource(mPid, resources);
163}
164
165// static
166sp<MediaCodec> MediaCodec::CreateByType(
167        const sp<ALooper> &looper, const char *mime, bool encoder, status_t *err, pid_t pid) {
168    sp<MediaCodec> codec = new MediaCodec(looper, pid);
169
170    const status_t ret = codec->init(mime, true /* nameIsType */, encoder);
171    if (err != NULL) {
172        *err = ret;
173    }
174    return ret == OK ? codec : NULL; // NULL deallocates codec.
175}
176
177// static
178sp<MediaCodec> MediaCodec::CreateByComponentName(
179        const sp<ALooper> &looper, const char *name, status_t *err, pid_t pid) {
180    sp<MediaCodec> codec = new MediaCodec(looper, pid);
181
182    const status_t ret = codec->init(name, false /* nameIsType */, false /* encoder */);
183    if (err != NULL) {
184        *err = ret;
185    }
186    return ret == OK ? codec : NULL; // NULL deallocates codec.
187}
188
189// static
190sp<PersistentSurface> MediaCodec::CreatePersistentInputSurface() {
191    OMXClient client;
192    CHECK_EQ(client.connect(), (status_t)OK);
193    sp<IOMX> omx = client.interface();
194
195    const sp<IMediaCodecList> mediaCodecList = MediaCodecList::getInstance();
196    if (mediaCodecList == NULL) {
197        ALOGE("Failed to obtain MediaCodecList!");
198        return NULL; // if called from Java should raise IOException
199    }
200
201    AString tmp;
202    sp<AMessage> globalSettings = mediaCodecList->getGlobalSettings();
203    if (globalSettings == NULL || !globalSettings->findString(
204            kMaxEncoderInputBuffers, &tmp)) {
205        ALOGE("Failed to get encoder input buffer count!");
206        return NULL;
207    }
208
209    int32_t bufferCount = strtol(tmp.c_str(), NULL, 10);
210    if (bufferCount <= 0
211            || bufferCount > BufferQueue::MAX_MAX_ACQUIRED_BUFFERS) {
212        ALOGE("Encoder input buffer count is invalid!");
213        return NULL;
214    }
215
216    sp<IGraphicBufferProducer> bufferProducer;
217    sp<IGraphicBufferConsumer> bufferConsumer;
218
219    status_t err = omx->createPersistentInputSurface(
220            &bufferProducer, &bufferConsumer);
221
222    if (err != OK) {
223        ALOGE("Failed to create persistent input surface.");
224        return NULL;
225    }
226
227    err = bufferConsumer->setMaxAcquiredBufferCount(bufferCount);
228
229    if (err != NO_ERROR) {
230        ALOGE("Unable to set BQ max acquired buffer count to %u: %d",
231                bufferCount, err);
232        return NULL;
233    }
234
235    return new PersistentSurface(bufferProducer, bufferConsumer);
236}
237
238MediaCodec::MediaCodec(const sp<ALooper> &looper, pid_t pid)
239    : mState(UNINITIALIZED),
240      mReleasedByResourceManager(false),
241      mLooper(looper),
242      mCodec(NULL),
243      mReplyID(0),
244      mFlags(0),
245      mStickyError(OK),
246      mSoftRenderer(NULL),
247      mResourceManagerClient(new ResourceManagerClient(this)),
248      mResourceManagerService(new ResourceManagerServiceProxy(pid)),
249      mBatteryStatNotified(false),
250      mIsVideo(false),
251      mVideoWidth(0),
252      mVideoHeight(0),
253      mRotationDegrees(0),
254      mDequeueInputTimeoutGeneration(0),
255      mDequeueInputReplyID(0),
256      mDequeueOutputTimeoutGeneration(0),
257      mDequeueOutputReplyID(0),
258      mHaveInputSurface(false),
259      mHavePendingInputBuffers(false) {
260}
261
262MediaCodec::~MediaCodec() {
263    CHECK_EQ(mState, UNINITIALIZED);
264    mResourceManagerService->removeResource(getId(mResourceManagerClient));
265}
266
267// static
268status_t MediaCodec::PostAndAwaitResponse(
269        const sp<AMessage> &msg, sp<AMessage> *response) {
270    status_t err = msg->postAndAwaitResponse(response);
271
272    if (err != OK) {
273        return err;
274    }
275
276    if (!(*response)->findInt32("err", &err)) {
277        err = OK;
278    }
279
280    return err;
281}
282
283void MediaCodec::PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err) {
284    int32_t finalErr = err;
285    if (mReleasedByResourceManager) {
286        // override the err code if MediaCodec has been released by ResourceManager.
287        finalErr = DEAD_OBJECT;
288    }
289
290    sp<AMessage> response = new AMessage;
291    response->setInt32("err", finalErr);
292    response->postReply(replyID);
293}
294
295status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) {
296    mResourceManagerService->init();
297
298    // save init parameters for reset
299    mInitName = name;
300    mInitNameIsType = nameIsType;
301    mInitIsEncoder = encoder;
302
303    // Current video decoders do not return from OMX_FillThisBuffer
304    // quickly, violating the OpenMAX specs, until that is remedied
305    // we need to invest in an extra looper to free the main event
306    // queue.
307
308    if (nameIsType || !strncasecmp(name.c_str(), "omx.", 4)) {
309        mCodec = new ACodec;
310    } else if (!nameIsType
311            && !strncasecmp(name.c_str(), "android.filter.", 15)) {
312        mCodec = new MediaFilter;
313    } else {
314        return NAME_NOT_FOUND;
315    }
316
317    bool secureCodec = false;
318    if (nameIsType && !strncasecmp(name.c_str(), "video/", 6)) {
319        mIsVideo = true;
320    } else {
321        AString tmp = name;
322        if (tmp.endsWith(".secure")) {
323            secureCodec = true;
324            tmp.erase(tmp.size() - 7, 7);
325        }
326        const sp<IMediaCodecList> mcl = MediaCodecList::getInstance();
327        if (mcl == NULL) {
328            mCodec = NULL;  // remove the codec.
329            return NO_INIT; // if called from Java should raise IOException
330        }
331        ssize_t codecIdx = mcl->findCodecByName(tmp.c_str());
332        if (codecIdx >= 0) {
333            const sp<MediaCodecInfo> info = mcl->getCodecInfo(codecIdx);
334            Vector<AString> mimes;
335            info->getSupportedMimes(&mimes);
336            for (size_t i = 0; i < mimes.size(); i++) {
337                if (mimes[i].startsWith("video/")) {
338                    mIsVideo = true;
339                    break;
340                }
341            }
342        }
343    }
344
345    if (mIsVideo) {
346        // video codec needs dedicated looper
347        if (mCodecLooper == NULL) {
348            mCodecLooper = new ALooper;
349            mCodecLooper->setName("CodecLooper");
350            mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
351        }
352
353        mCodecLooper->registerHandler(mCodec);
354    } else {
355        mLooper->registerHandler(mCodec);
356    }
357
358    mLooper->registerHandler(this);
359
360    mCodec->setNotificationMessage(new AMessage(kWhatCodecNotify, this));
361
362    sp<AMessage> msg = new AMessage(kWhatInit, this);
363    msg->setString("name", name);
364    msg->setInt32("nameIsType", nameIsType);
365
366    if (nameIsType) {
367        msg->setInt32("encoder", encoder);
368    }
369
370    status_t err;
371    Vector<MediaResource> resources;
372    const char *type = secureCodec ? kResourceSecureCodec : kResourceNonSecureCodec;
373    const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec;
374    resources.push_back(MediaResource(String8(type), String8(subtype), 1));
375    for (int i = 0; i <= kMaxRetry; ++i) {
376        if (i > 0) {
377            // Don't try to reclaim resource for the first time.
378            if (!mResourceManagerService->reclaimResource(resources)) {
379                break;
380            }
381        }
382
383        sp<AMessage> response;
384        err = PostAndAwaitResponse(msg, &response);
385        if (!isResourceError(err)) {
386            break;
387        }
388    }
389    return err;
390}
391
392status_t MediaCodec::setCallback(const sp<AMessage> &callback) {
393    sp<AMessage> msg = new AMessage(kWhatSetCallback, this);
394    msg->setMessage("callback", callback);
395
396    sp<AMessage> response;
397    return PostAndAwaitResponse(msg, &response);
398}
399
400status_t MediaCodec::setOnFrameRenderedNotification(const sp<AMessage> &notify) {
401    sp<AMessage> msg = new AMessage(kWhatSetNotification, this);
402    msg->setMessage("on-frame-rendered", notify);
403    return msg->post();
404}
405
406status_t MediaCodec::configure(
407        const sp<AMessage> &format,
408        const sp<Surface> &surface,
409        const sp<ICrypto> &crypto,
410        uint32_t flags) {
411    sp<AMessage> msg = new AMessage(kWhatConfigure, this);
412
413    if (mIsVideo) {
414        format->findInt32("width", &mVideoWidth);
415        format->findInt32("height", &mVideoHeight);
416        if (!format->findInt32("rotation-degrees", &mRotationDegrees)) {
417            mRotationDegrees = 0;
418        }
419    }
420
421    msg->setMessage("format", format);
422    msg->setInt32("flags", flags);
423    msg->setObject("surface", surface);
424
425    if (crypto != NULL) {
426        msg->setPointer("crypto", crypto.get());
427    }
428
429    // save msg for reset
430    mConfigureMsg = msg;
431
432    status_t err;
433    Vector<MediaResource> resources;
434    const char *type = (mFlags & kFlagIsSecure) ?
435            kResourceSecureCodec : kResourceNonSecureCodec;
436    const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec;
437    resources.push_back(MediaResource(String8(type), String8(subtype), 1));
438    // Don't know the buffer size at this point, but it's fine to use 1 because
439    // the reclaimResource call doesn't consider the requester's buffer size for now.
440    resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1));
441    for (int i = 0; i <= kMaxRetry; ++i) {
442        if (i > 0) {
443            // Don't try to reclaim resource for the first time.
444            if (!mResourceManagerService->reclaimResource(resources)) {
445                break;
446            }
447        }
448
449        sp<AMessage> response;
450        err = PostAndAwaitResponse(msg, &response);
451        if (err != OK && err != INVALID_OPERATION) {
452            // MediaCodec now set state to UNINITIALIZED upon any fatal error.
453            // To maintain backward-compatibility, do a reset() to put codec
454            // back into INITIALIZED state.
455            // But don't reset if the err is INVALID_OPERATION, which means
456            // the configure failure is due to wrong state.
457
458            ALOGE("configure failed with err 0x%08x, resetting...", err);
459            reset();
460        }
461        if (!isResourceError(err)) {
462            break;
463        }
464    }
465    return err;
466}
467
468status_t MediaCodec::setInputSurface(
469        const sp<PersistentSurface> &surface) {
470    sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this);
471    msg->setObject("input-surface", surface.get());
472
473    sp<AMessage> response;
474    return PostAndAwaitResponse(msg, &response);
475}
476
477status_t MediaCodec::setSurface(const sp<Surface> &surface) {
478    sp<AMessage> msg = new AMessage(kWhatSetSurface, this);
479    msg->setObject("surface", surface);
480
481    sp<AMessage> response;
482    return PostAndAwaitResponse(msg, &response);
483}
484
485status_t MediaCodec::createInputSurface(
486        sp<IGraphicBufferProducer>* bufferProducer) {
487    sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, this);
488
489    sp<AMessage> response;
490    status_t err = PostAndAwaitResponse(msg, &response);
491    if (err == NO_ERROR) {
492        // unwrap the sp<IGraphicBufferProducer>
493        sp<RefBase> obj;
494        bool found = response->findObject("input-surface", &obj);
495        CHECK(found);
496        sp<BufferProducerWrapper> wrapper(
497                static_cast<BufferProducerWrapper*>(obj.get()));
498        *bufferProducer = wrapper->getBufferProducer();
499    } else {
500        ALOGW("createInputSurface failed, err=%d", err);
501    }
502    return err;
503}
504
505uint64_t MediaCodec::getGraphicBufferSize() {
506    if (!mIsVideo) {
507        return 0;
508    }
509
510    uint64_t size = 0;
511    size_t portNum = sizeof(mPortBuffers) / sizeof((mPortBuffers)[0]);
512    for (size_t i = 0; i < portNum; ++i) {
513        // TODO: this is just an estimation, we should get the real buffer size from ACodec.
514        size += mPortBuffers[i].size() * mVideoWidth * mVideoHeight * 3 / 2;
515    }
516    return size;
517}
518
519void MediaCodec::addResource(const String8 &type, const String8 &subtype, uint64_t value) {
520    Vector<MediaResource> resources;
521    resources.push_back(MediaResource(type, subtype, value));
522    mResourceManagerService->addResource(
523            getId(mResourceManagerClient), mResourceManagerClient, resources);
524}
525
526status_t MediaCodec::start() {
527    sp<AMessage> msg = new AMessage(kWhatStart, this);
528
529    status_t err;
530    Vector<MediaResource> resources;
531    const char *type = (mFlags & kFlagIsSecure) ?
532            kResourceSecureCodec : kResourceNonSecureCodec;
533    const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec;
534    resources.push_back(MediaResource(String8(type), String8(subtype), 1));
535    // Don't know the buffer size at this point, but it's fine to use 1 because
536    // the reclaimResource call doesn't consider the requester's buffer size for now.
537    resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1));
538    for (int i = 0; i <= kMaxRetry; ++i) {
539        if (i > 0) {
540            // Don't try to reclaim resource for the first time.
541            if (!mResourceManagerService->reclaimResource(resources)) {
542                break;
543            }
544            // Recover codec from previous error before retry start.
545            err = reset();
546            if (err != OK) {
547                ALOGE("retrying start: failed to reset codec");
548                break;
549            }
550            sp<AMessage> response;
551            err = PostAndAwaitResponse(mConfigureMsg, &response);
552            if (err != OK) {
553                ALOGE("retrying start: failed to configure codec");
554                break;
555            }
556        }
557
558        sp<AMessage> response;
559        err = PostAndAwaitResponse(msg, &response);
560        if (!isResourceError(err)) {
561            break;
562        }
563    }
564    return err;
565}
566
567status_t MediaCodec::stop() {
568    sp<AMessage> msg = new AMessage(kWhatStop, this);
569
570    sp<AMessage> response;
571    return PostAndAwaitResponse(msg, &response);
572}
573
574status_t MediaCodec::reclaim() {
575    ALOGD("MediaCodec::reclaim(%p) %s", this, mInitName.c_str());
576    sp<AMessage> msg = new AMessage(kWhatRelease, this);
577    msg->setInt32("reclaimed", 1);
578
579    sp<AMessage> response;
580    return PostAndAwaitResponse(msg, &response);
581}
582
583status_t MediaCodec::release() {
584    sp<AMessage> msg = new AMessage(kWhatRelease, this);
585
586    sp<AMessage> response;
587    return PostAndAwaitResponse(msg, &response);
588}
589
590status_t MediaCodec::reset() {
591    /* When external-facing MediaCodec object is created,
592       it is already initialized.  Thus, reset is essentially
593       release() followed by init(), plus clearing the state */
594
595    status_t err = release();
596
597    // unregister handlers
598    if (mCodec != NULL) {
599        if (mCodecLooper != NULL) {
600            mCodecLooper->unregisterHandler(mCodec->id());
601        } else {
602            mLooper->unregisterHandler(mCodec->id());
603        }
604        mCodec = NULL;
605    }
606    mLooper->unregisterHandler(id());
607
608    mFlags = 0;    // clear all flags
609    mStickyError = OK;
610
611    // reset state not reset by setState(UNINITIALIZED)
612    mReplyID = 0;
613    mDequeueInputReplyID = 0;
614    mDequeueOutputReplyID = 0;
615    mDequeueInputTimeoutGeneration = 0;
616    mDequeueOutputTimeoutGeneration = 0;
617    mHaveInputSurface = false;
618
619    if (err == OK) {
620        err = init(mInitName, mInitNameIsType, mInitIsEncoder);
621    }
622    return err;
623}
624
625status_t MediaCodec::queueInputBuffer(
626        size_t index,
627        size_t offset,
628        size_t size,
629        int64_t presentationTimeUs,
630        uint32_t flags,
631        AString *errorDetailMsg) {
632    if (errorDetailMsg != NULL) {
633        errorDetailMsg->clear();
634    }
635
636    sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
637    msg->setSize("index", index);
638    msg->setSize("offset", offset);
639    msg->setSize("size", size);
640    msg->setInt64("timeUs", presentationTimeUs);
641    msg->setInt32("flags", flags);
642    msg->setPointer("errorDetailMsg", errorDetailMsg);
643
644    sp<AMessage> response;
645    return PostAndAwaitResponse(msg, &response);
646}
647
648status_t MediaCodec::queueSecureInputBuffer(
649        size_t index,
650        size_t offset,
651        const CryptoPlugin::SubSample *subSamples,
652        size_t numSubSamples,
653        const uint8_t key[16],
654        const uint8_t iv[16],
655        CryptoPlugin::Mode mode,
656        int64_t presentationTimeUs,
657        uint32_t flags,
658        AString *errorDetailMsg) {
659    if (errorDetailMsg != NULL) {
660        errorDetailMsg->clear();
661    }
662
663    sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
664    msg->setSize("index", index);
665    msg->setSize("offset", offset);
666    msg->setPointer("subSamples", (void *)subSamples);
667    msg->setSize("numSubSamples", numSubSamples);
668    msg->setPointer("key", (void *)key);
669    msg->setPointer("iv", (void *)iv);
670    msg->setInt32("mode", mode);
671    msg->setInt64("timeUs", presentationTimeUs);
672    msg->setInt32("flags", flags);
673    msg->setPointer("errorDetailMsg", errorDetailMsg);
674
675    sp<AMessage> response;
676    status_t err = PostAndAwaitResponse(msg, &response);
677
678    return err;
679}
680
681status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) {
682    sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, this);
683    msg->setInt64("timeoutUs", timeoutUs);
684
685    sp<AMessage> response;
686    status_t err;
687    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
688        return err;
689    }
690
691    CHECK(response->findSize("index", index));
692
693    return OK;
694}
695
696status_t MediaCodec::dequeueOutputBuffer(
697        size_t *index,
698        size_t *offset,
699        size_t *size,
700        int64_t *presentationTimeUs,
701        uint32_t *flags,
702        int64_t timeoutUs) {
703    sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, this);
704    msg->setInt64("timeoutUs", timeoutUs);
705
706    sp<AMessage> response;
707    status_t err;
708    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
709        return err;
710    }
711
712    CHECK(response->findSize("index", index));
713    CHECK(response->findSize("offset", offset));
714    CHECK(response->findSize("size", size));
715    CHECK(response->findInt64("timeUs", presentationTimeUs));
716    CHECK(response->findInt32("flags", (int32_t *)flags));
717
718    return OK;
719}
720
721status_t MediaCodec::renderOutputBufferAndRelease(size_t index) {
722    sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
723    msg->setSize("index", index);
724    msg->setInt32("render", true);
725
726    sp<AMessage> response;
727    return PostAndAwaitResponse(msg, &response);
728}
729
730status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) {
731    sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
732    msg->setSize("index", index);
733    msg->setInt32("render", true);
734    msg->setInt64("timestampNs", timestampNs);
735
736    sp<AMessage> response;
737    return PostAndAwaitResponse(msg, &response);
738}
739
740status_t MediaCodec::releaseOutputBuffer(size_t index) {
741    sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
742    msg->setSize("index", index);
743
744    sp<AMessage> response;
745    return PostAndAwaitResponse(msg, &response);
746}
747
748status_t MediaCodec::signalEndOfInputStream() {
749    sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, this);
750
751    sp<AMessage> response;
752    return PostAndAwaitResponse(msg, &response);
753}
754
755status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const {
756    sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, this);
757
758    sp<AMessage> response;
759    status_t err;
760    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
761        return err;
762    }
763
764    CHECK(response->findMessage("format", format));
765
766    return OK;
767}
768
769status_t MediaCodec::getInputFormat(sp<AMessage> *format) const {
770    sp<AMessage> msg = new AMessage(kWhatGetInputFormat, this);
771
772    sp<AMessage> response;
773    status_t err;
774    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
775        return err;
776    }
777
778    CHECK(response->findMessage("format", format));
779
780    return OK;
781}
782
783status_t MediaCodec::getName(AString *name) const {
784    sp<AMessage> msg = new AMessage(kWhatGetName, this);
785
786    sp<AMessage> response;
787    status_t err;
788    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
789        return err;
790    }
791
792    CHECK(response->findString("name", name));
793
794    return OK;
795}
796
797status_t MediaCodec::getWidevineLegacyBuffers(Vector<sp<ABuffer> > *buffers) const {
798    sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
799    msg->setInt32("portIndex", kPortIndexInput);
800    msg->setPointer("buffers", buffers);
801    msg->setInt32("widevine", true);
802
803    sp<AMessage> response;
804    return PostAndAwaitResponse(msg, &response);
805}
806
807status_t MediaCodec::getInputBuffers(Vector<sp<ABuffer> > *buffers) const {
808    sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
809    msg->setInt32("portIndex", kPortIndexInput);
810    msg->setPointer("buffers", buffers);
811
812    sp<AMessage> response;
813    return PostAndAwaitResponse(msg, &response);
814}
815
816status_t MediaCodec::getOutputBuffers(Vector<sp<ABuffer> > *buffers) const {
817    sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
818    msg->setInt32("portIndex", kPortIndexOutput);
819    msg->setPointer("buffers", buffers);
820
821    sp<AMessage> response;
822    return PostAndAwaitResponse(msg, &response);
823}
824
825status_t MediaCodec::getOutputBuffer(size_t index, sp<ABuffer> *buffer) {
826    sp<AMessage> format;
827    return getBufferAndFormat(kPortIndexOutput, index, buffer, &format);
828}
829
830status_t MediaCodec::getOutputFormat(size_t index, sp<AMessage> *format) {
831    sp<ABuffer> buffer;
832    return getBufferAndFormat(kPortIndexOutput, index, &buffer, format);
833}
834
835status_t MediaCodec::getInputBuffer(size_t index, sp<ABuffer> *buffer) {
836    sp<AMessage> format;
837    return getBufferAndFormat(kPortIndexInput, index, buffer, &format);
838}
839
840bool MediaCodec::isExecuting() const {
841    return mState == STARTED || mState == FLUSHED;
842}
843
844status_t MediaCodec::getBufferAndFormat(
845        size_t portIndex, size_t index,
846        sp<ABuffer> *buffer, sp<AMessage> *format) {
847    // use mutex instead of a context switch
848
849    if (mReleasedByResourceManager) {
850        return DEAD_OBJECT;
851    }
852
853    buffer->clear();
854    format->clear();
855    if (!isExecuting()) {
856        return INVALID_OPERATION;
857    }
858
859    // we do not want mPortBuffers to change during this section
860    // we also don't want mOwnedByClient to change during this
861    Mutex::Autolock al(mBufferLock);
862    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
863    if (index < buffers->size()) {
864        const BufferInfo &info = buffers->itemAt(index);
865        if (info.mOwnedByClient) {
866            // by the time buffers array is initialized, crypto is set
867            if (portIndex == kPortIndexInput && mCrypto != NULL) {
868                *buffer = info.mEncryptedData;
869            } else {
870                *buffer = info.mData;
871            }
872            *format = info.mFormat;
873        }
874    }
875    return OK;
876}
877
878status_t MediaCodec::flush() {
879    sp<AMessage> msg = new AMessage(kWhatFlush, this);
880
881    sp<AMessage> response;
882    return PostAndAwaitResponse(msg, &response);
883}
884
885status_t MediaCodec::requestIDRFrame() {
886    (new AMessage(kWhatRequestIDRFrame, this))->post();
887
888    return OK;
889}
890
891void MediaCodec::requestActivityNotification(const sp<AMessage> &notify) {
892    sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, this);
893    msg->setMessage("notify", notify);
894    msg->post();
895}
896
897////////////////////////////////////////////////////////////////////////////////
898
899void MediaCodec::cancelPendingDequeueOperations() {
900    if (mFlags & kFlagDequeueInputPending) {
901        PostReplyWithError(mDequeueInputReplyID, INVALID_OPERATION);
902
903        ++mDequeueInputTimeoutGeneration;
904        mDequeueInputReplyID = 0;
905        mFlags &= ~kFlagDequeueInputPending;
906    }
907
908    if (mFlags & kFlagDequeueOutputPending) {
909        PostReplyWithError(mDequeueOutputReplyID, INVALID_OPERATION);
910
911        ++mDequeueOutputTimeoutGeneration;
912        mDequeueOutputReplyID = 0;
913        mFlags &= ~kFlagDequeueOutputPending;
914    }
915}
916
917bool MediaCodec::handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest) {
918    if (!isExecuting() || (mFlags & kFlagIsAsync)
919            || (newRequest && (mFlags & kFlagDequeueInputPending))) {
920        PostReplyWithError(replyID, INVALID_OPERATION);
921        return true;
922    } else if (mFlags & kFlagStickyError) {
923        PostReplyWithError(replyID, getStickyError());
924        return true;
925    }
926
927    ssize_t index = dequeuePortBuffer(kPortIndexInput);
928
929    if (index < 0) {
930        CHECK_EQ(index, -EAGAIN);
931        return false;
932    }
933
934    sp<AMessage> response = new AMessage;
935    response->setSize("index", index);
936    response->postReply(replyID);
937
938    return true;
939}
940
941bool MediaCodec::handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest) {
942    if (!isExecuting() || (mFlags & kFlagIsAsync)
943            || (newRequest && (mFlags & kFlagDequeueOutputPending))) {
944        PostReplyWithError(replyID, INVALID_OPERATION);
945    } else if (mFlags & kFlagStickyError) {
946        PostReplyWithError(replyID, getStickyError());
947    } else if (mFlags & kFlagOutputBuffersChanged) {
948        PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED);
949        mFlags &= ~kFlagOutputBuffersChanged;
950    } else if (mFlags & kFlagOutputFormatChanged) {
951        PostReplyWithError(replyID, INFO_FORMAT_CHANGED);
952        mFlags &= ~kFlagOutputFormatChanged;
953    } else {
954        sp<AMessage> response = new AMessage;
955        ssize_t index = dequeuePortBuffer(kPortIndexOutput);
956
957        if (index < 0) {
958            CHECK_EQ(index, -EAGAIN);
959            return false;
960        }
961
962        const sp<ABuffer> &buffer =
963            mPortBuffers[kPortIndexOutput].itemAt(index).mData;
964
965        response->setSize("index", index);
966        response->setSize("offset", buffer->offset());
967        response->setSize("size", buffer->size());
968
969        int64_t timeUs;
970        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
971
972        response->setInt64("timeUs", timeUs);
973
974        int32_t omxFlags;
975        CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags));
976
977        uint32_t flags = 0;
978        if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) {
979            flags |= BUFFER_FLAG_SYNCFRAME;
980        }
981        if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) {
982            flags |= BUFFER_FLAG_CODECCONFIG;
983        }
984        if (omxFlags & OMX_BUFFERFLAG_EOS) {
985            flags |= BUFFER_FLAG_EOS;
986        }
987
988        response->setInt32("flags", flags);
989        response->postReply(replyID);
990    }
991
992    return true;
993}
994
995void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
996    switch (msg->what()) {
997        case kWhatCodecNotify:
998        {
999            int32_t what;
1000            CHECK(msg->findInt32("what", &what));
1001
1002            switch (what) {
1003                case CodecBase::kWhatError:
1004                {
1005                    int32_t err, actionCode;
1006                    CHECK(msg->findInt32("err", &err));
1007                    CHECK(msg->findInt32("actionCode", &actionCode));
1008
1009                    ALOGE("Codec reported err %#x, actionCode %d, while in state %d",
1010                            err, actionCode, mState);
1011                    if (err == DEAD_OBJECT) {
1012                        mFlags |= kFlagSawMediaServerDie;
1013                        mFlags &= ~kFlagIsComponentAllocated;
1014                    }
1015
1016                    bool sendErrorResponse = true;
1017
1018                    switch (mState) {
1019                        case INITIALIZING:
1020                        {
1021                            setState(UNINITIALIZED);
1022                            break;
1023                        }
1024
1025                        case CONFIGURING:
1026                        {
1027                            setState(actionCode == ACTION_CODE_FATAL ?
1028                                    UNINITIALIZED : INITIALIZED);
1029                            break;
1030                        }
1031
1032                        case STARTING:
1033                        {
1034                            setState(actionCode == ACTION_CODE_FATAL ?
1035                                    UNINITIALIZED : CONFIGURED);
1036                            break;
1037                        }
1038
1039                        case STOPPING:
1040                        case RELEASING:
1041                        {
1042                            // Ignore the error, assuming we'll still get
1043                            // the shutdown complete notification.
1044
1045                            sendErrorResponse = false;
1046
1047                            if (mFlags & kFlagSawMediaServerDie) {
1048                                // MediaServer died, there definitely won't
1049                                // be a shutdown complete notification after
1050                                // all.
1051
1052                                // note that we're directly going from
1053                                // STOPPING->UNINITIALIZED, instead of the
1054                                // usual STOPPING->INITIALIZED state.
1055                                setState(UNINITIALIZED);
1056                                if (mState == RELEASING) {
1057                                    mComponentName.clear();
1058                                }
1059                                (new AMessage)->postReply(mReplyID);
1060                            }
1061                            break;
1062                        }
1063
1064                        case FLUSHING:
1065                        {
1066                            if (actionCode == ACTION_CODE_FATAL) {
1067                                setState(UNINITIALIZED);
1068                            } else {
1069                                setState(
1070                                        (mFlags & kFlagIsAsync) ? FLUSHED : STARTED);
1071                            }
1072                            break;
1073                        }
1074
1075                        case FLUSHED:
1076                        case STARTED:
1077                        {
1078                            sendErrorResponse = false;
1079
1080                            setStickyError(err);
1081                            postActivityNotificationIfPossible();
1082
1083                            cancelPendingDequeueOperations();
1084
1085                            if (mFlags & kFlagIsAsync) {
1086                                onError(err, actionCode);
1087                            }
1088                            switch (actionCode) {
1089                            case ACTION_CODE_TRANSIENT:
1090                                break;
1091                            case ACTION_CODE_RECOVERABLE:
1092                                setState(INITIALIZED);
1093                                break;
1094                            default:
1095                                setState(UNINITIALIZED);
1096                                break;
1097                            }
1098                            break;
1099                        }
1100
1101                        default:
1102                        {
1103                            sendErrorResponse = false;
1104
1105                            setStickyError(err);
1106                            postActivityNotificationIfPossible();
1107
1108                            // actionCode in an uninitialized state is always fatal.
1109                            if (mState == UNINITIALIZED) {
1110                                actionCode = ACTION_CODE_FATAL;
1111                            }
1112                            if (mFlags & kFlagIsAsync) {
1113                                onError(err, actionCode);
1114                            }
1115                            switch (actionCode) {
1116                            case ACTION_CODE_TRANSIENT:
1117                                break;
1118                            case ACTION_CODE_RECOVERABLE:
1119                                setState(INITIALIZED);
1120                                break;
1121                            default:
1122                                setState(UNINITIALIZED);
1123                                break;
1124                            }
1125                            break;
1126                        }
1127                    }
1128
1129                    if (sendErrorResponse) {
1130                        PostReplyWithError(mReplyID, err);
1131                    }
1132                    break;
1133                }
1134
1135                case CodecBase::kWhatComponentAllocated:
1136                {
1137                    CHECK_EQ(mState, INITIALIZING);
1138                    setState(INITIALIZED);
1139                    mFlags |= kFlagIsComponentAllocated;
1140
1141                    CHECK(msg->findString("componentName", &mComponentName));
1142
1143                    if (mComponentName.startsWith("OMX.google.")) {
1144                        mFlags |= kFlagUsesSoftwareRenderer;
1145                    } else {
1146                        mFlags &= ~kFlagUsesSoftwareRenderer;
1147                    }
1148
1149                    String8 resourceType;
1150                    if (mComponentName.endsWith(".secure")) {
1151                        mFlags |= kFlagIsSecure;
1152                        resourceType = String8(kResourceSecureCodec);
1153                    } else {
1154                        mFlags &= ~kFlagIsSecure;
1155                        resourceType = String8(kResourceNonSecureCodec);
1156                    }
1157
1158                    if (mIsVideo) {
1159                        // audio codec is currently ignored.
1160                        addResource(resourceType, String8(kResourceVideoCodec), 1);
1161                    }
1162
1163                    (new AMessage)->postReply(mReplyID);
1164                    break;
1165                }
1166
1167                case CodecBase::kWhatComponentConfigured:
1168                {
1169                    if (mState == UNINITIALIZED || mState == INITIALIZED) {
1170                        // In case a kWhatError message came in and replied with error,
1171                        // we log a warning and ignore.
1172                        ALOGW("configure interrupted by error, current state %d", mState);
1173                        break;
1174                    }
1175                    CHECK_EQ(mState, CONFIGURING);
1176
1177                    // reset input surface flag
1178                    mHaveInputSurface = false;
1179
1180                    CHECK(msg->findMessage("input-format", &mInputFormat));
1181                    CHECK(msg->findMessage("output-format", &mOutputFormat));
1182
1183                    int32_t usingSwRenderer;
1184                    if (mOutputFormat->findInt32("using-sw-renderer", &usingSwRenderer)
1185                            && usingSwRenderer) {
1186                        mFlags |= kFlagUsesSoftwareRenderer;
1187                    }
1188                    setState(CONFIGURED);
1189                    (new AMessage)->postReply(mReplyID);
1190                    break;
1191                }
1192
1193                case CodecBase::kWhatInputSurfaceCreated:
1194                {
1195                    // response to initiateCreateInputSurface()
1196                    status_t err = NO_ERROR;
1197                    sp<AMessage> response = new AMessage;
1198                    if (!msg->findInt32("err", &err)) {
1199                        sp<RefBase> obj;
1200                        msg->findObject("input-surface", &obj);
1201                        CHECK(obj != NULL);
1202                        response->setObject("input-surface", obj);
1203                        mHaveInputSurface = true;
1204                    } else {
1205                        response->setInt32("err", err);
1206                    }
1207                    response->postReply(mReplyID);
1208                    break;
1209                }
1210
1211                case CodecBase::kWhatInputSurfaceAccepted:
1212                {
1213                    // response to initiateSetInputSurface()
1214                    status_t err = NO_ERROR;
1215                    sp<AMessage> response = new AMessage();
1216                    if (!msg->findInt32("err", &err)) {
1217                        mHaveInputSurface = true;
1218                    } else {
1219                        response->setInt32("err", err);
1220                    }
1221                    response->postReply(mReplyID);
1222                    break;
1223                }
1224
1225                case CodecBase::kWhatSignaledInputEOS:
1226                {
1227                    // response to signalEndOfInputStream()
1228                    sp<AMessage> response = new AMessage;
1229                    status_t err;
1230                    if (msg->findInt32("err", &err)) {
1231                        response->setInt32("err", err);
1232                    }
1233                    response->postReply(mReplyID);
1234                    break;
1235                }
1236
1237
1238                case CodecBase::kWhatBuffersAllocated:
1239                {
1240                    Mutex::Autolock al(mBufferLock);
1241                    int32_t portIndex;
1242                    CHECK(msg->findInt32("portIndex", &portIndex));
1243
1244                    ALOGV("%s buffers allocated",
1245                          portIndex == kPortIndexInput ? "input" : "output");
1246
1247                    CHECK(portIndex == kPortIndexInput
1248                            || portIndex == kPortIndexOutput);
1249
1250                    mPortBuffers[portIndex].clear();
1251
1252                    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
1253
1254                    sp<RefBase> obj;
1255                    CHECK(msg->findObject("portDesc", &obj));
1256
1257                    sp<CodecBase::PortDescription> portDesc =
1258                        static_cast<CodecBase::PortDescription *>(obj.get());
1259
1260                    size_t numBuffers = portDesc->countBuffers();
1261
1262                    size_t totalSize = 0;
1263                    for (size_t i = 0; i < numBuffers; ++i) {
1264                        if (portIndex == kPortIndexInput && mCrypto != NULL) {
1265                            totalSize += portDesc->bufferAt(i)->capacity();
1266                        }
1267                    }
1268
1269                    if (totalSize) {
1270                        mDealer = new MemoryDealer(totalSize, "MediaCodec");
1271                    }
1272
1273                    for (size_t i = 0; i < numBuffers; ++i) {
1274                        BufferInfo info;
1275                        info.mBufferID = portDesc->bufferIDAt(i);
1276                        info.mOwnedByClient = false;
1277                        info.mData = portDesc->bufferAt(i);
1278
1279                        if (portIndex == kPortIndexInput && mCrypto != NULL) {
1280                            sp<IMemory> mem = mDealer->allocate(info.mData->capacity());
1281                            info.mEncryptedData =
1282                                new ABuffer(mem->pointer(), info.mData->capacity());
1283                            info.mSharedEncryptedBuffer = mem;
1284                        }
1285
1286                        buffers->push_back(info);
1287                    }
1288
1289                    if (portIndex == kPortIndexOutput) {
1290                        if (mState == STARTING) {
1291                            // We're always allocating output buffers after
1292                            // allocating input buffers, so this is a good
1293                            // indication that now all buffers are allocated.
1294                            if (mIsVideo) {
1295                                String8 subtype;
1296                                addResource(
1297                                        String8(kResourceGraphicMemory),
1298                                        subtype,
1299                                        getGraphicBufferSize());
1300                            }
1301                            setState(STARTED);
1302                            (new AMessage)->postReply(mReplyID);
1303                        } else {
1304                            mFlags |= kFlagOutputBuffersChanged;
1305                            postActivityNotificationIfPossible();
1306                        }
1307                    }
1308                    break;
1309                }
1310
1311                case CodecBase::kWhatOutputFormatChanged:
1312                {
1313                    ALOGV("codec output format changed");
1314
1315                    if (mSoftRenderer == NULL &&
1316                            mSurface != NULL &&
1317                            (mFlags & kFlagUsesSoftwareRenderer)) {
1318                        AString mime;
1319                        CHECK(msg->findString("mime", &mime));
1320
1321                        if (mime.startsWithIgnoreCase("video/")) {
1322                            mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees);
1323                        }
1324                    }
1325
1326                    mOutputFormat = msg;
1327
1328                    if (mFlags & kFlagIsEncoder) {
1329                        // Before we announce the format change we should
1330                        // collect codec specific data and amend the output
1331                        // format as necessary.
1332                        mFlags |= kFlagGatherCodecSpecificData;
1333                    } else if (mFlags & kFlagIsAsync) {
1334                        onOutputFormatChanged();
1335                    } else {
1336                        mFlags |= kFlagOutputFormatChanged;
1337                        postActivityNotificationIfPossible();
1338                    }
1339
1340                    // Notify mCrypto of video resolution changes
1341                    if (mCrypto != NULL) {
1342                        int32_t left, top, right, bottom, width, height;
1343                        if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
1344                            mCrypto->notifyResolution(right - left + 1, bottom - top + 1);
1345                        } else if (mOutputFormat->findInt32("width", &width)
1346                                && mOutputFormat->findInt32("height", &height)) {
1347                            mCrypto->notifyResolution(width, height);
1348                        }
1349                    }
1350
1351                    break;
1352                }
1353
1354                case CodecBase::kWhatOutputFramesRendered:
1355                {
1356                    // ignore these in all states except running, and check that we have a
1357                    // notification set
1358                    if (mState == STARTED && mOnFrameRenderedNotification != NULL) {
1359                        sp<AMessage> notify = mOnFrameRenderedNotification->dup();
1360                        notify->setMessage("data", msg);
1361                        notify->post();
1362                    }
1363                    break;
1364                }
1365
1366                case CodecBase::kWhatFillThisBuffer:
1367                {
1368                    /* size_t index = */updateBuffers(kPortIndexInput, msg);
1369
1370                    if (mState == FLUSHING
1371                            || mState == STOPPING
1372                            || mState == RELEASING) {
1373                        returnBuffersToCodecOnPort(kPortIndexInput);
1374                        break;
1375                    }
1376
1377                    if (!mCSD.empty()) {
1378                        ssize_t index = dequeuePortBuffer(kPortIndexInput);
1379                        CHECK_GE(index, 0);
1380
1381                        // If codec specific data had been specified as
1382                        // part of the format in the call to configure and
1383                        // if there's more csd left, we submit it here
1384                        // clients only get access to input buffers once
1385                        // this data has been exhausted.
1386
1387                        status_t err = queueCSDInputBuffer(index);
1388
1389                        if (err != OK) {
1390                            ALOGE("queueCSDInputBuffer failed w/ error %d",
1391                                  err);
1392
1393                            setStickyError(err);
1394                            postActivityNotificationIfPossible();
1395
1396                            cancelPendingDequeueOperations();
1397                        }
1398                        break;
1399                    }
1400
1401                    if (mFlags & kFlagIsAsync) {
1402                        if (!mHaveInputSurface) {
1403                            if (mState == FLUSHED) {
1404                                mHavePendingInputBuffers = true;
1405                            } else {
1406                                onInputBufferAvailable();
1407                            }
1408                        }
1409                    } else if (mFlags & kFlagDequeueInputPending) {
1410                        CHECK(handleDequeueInputBuffer(mDequeueInputReplyID));
1411
1412                        ++mDequeueInputTimeoutGeneration;
1413                        mFlags &= ~kFlagDequeueInputPending;
1414                        mDequeueInputReplyID = 0;
1415                    } else {
1416                        postActivityNotificationIfPossible();
1417                    }
1418                    break;
1419                }
1420
1421                case CodecBase::kWhatDrainThisBuffer:
1422                {
1423                    /* size_t index = */updateBuffers(kPortIndexOutput, msg);
1424
1425                    if (mState == FLUSHING
1426                            || mState == STOPPING
1427                            || mState == RELEASING) {
1428                        returnBuffersToCodecOnPort(kPortIndexOutput);
1429                        break;
1430                    }
1431
1432                    sp<ABuffer> buffer;
1433                    CHECK(msg->findBuffer("buffer", &buffer));
1434
1435                    int32_t omxFlags;
1436                    CHECK(msg->findInt32("flags", &omxFlags));
1437
1438                    buffer->meta()->setInt32("omxFlags", omxFlags);
1439
1440                    if (mFlags & kFlagGatherCodecSpecificData) {
1441                        // This is the very first output buffer after a
1442                        // format change was signalled, it'll either contain
1443                        // the one piece of codec specific data we can expect
1444                        // or there won't be codec specific data.
1445                        if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) {
1446                            status_t err =
1447                                amendOutputFormatWithCodecSpecificData(buffer);
1448
1449                            if (err != OK) {
1450                                ALOGE("Codec spit out malformed codec "
1451                                      "specific data!");
1452                            }
1453                        }
1454
1455                        mFlags &= ~kFlagGatherCodecSpecificData;
1456                        if (mFlags & kFlagIsAsync) {
1457                            onOutputFormatChanged();
1458                        } else {
1459                            mFlags |= kFlagOutputFormatChanged;
1460                        }
1461                    }
1462
1463                    if (mFlags & kFlagIsAsync) {
1464                        onOutputBufferAvailable();
1465                    } else if (mFlags & kFlagDequeueOutputPending) {
1466                        CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID));
1467
1468                        ++mDequeueOutputTimeoutGeneration;
1469                        mFlags &= ~kFlagDequeueOutputPending;
1470                        mDequeueOutputReplyID = 0;
1471                    } else {
1472                        postActivityNotificationIfPossible();
1473                    }
1474
1475                    break;
1476                }
1477
1478                case CodecBase::kWhatEOS:
1479                {
1480                    // We already notify the client of this by using the
1481                    // corresponding flag in "onOutputBufferReady".
1482                    break;
1483                }
1484
1485                case CodecBase::kWhatShutdownCompleted:
1486                {
1487                    if (mState == STOPPING) {
1488                        setState(INITIALIZED);
1489                    } else {
1490                        CHECK_EQ(mState, RELEASING);
1491                        setState(UNINITIALIZED);
1492                        mComponentName.clear();
1493                    }
1494                    mFlags &= ~kFlagIsComponentAllocated;
1495
1496                    mResourceManagerService->removeResource(getId(mResourceManagerClient));
1497
1498                    (new AMessage)->postReply(mReplyID);
1499                    break;
1500                }
1501
1502                case CodecBase::kWhatFlushCompleted:
1503                {
1504                    if (mState != FLUSHING) {
1505                        ALOGW("received FlushCompleted message in state %d",
1506                                mState);
1507                        break;
1508                    }
1509
1510                    if (mFlags & kFlagIsAsync) {
1511                        setState(FLUSHED);
1512                    } else {
1513                        setState(STARTED);
1514                        mCodec->signalResume();
1515                    }
1516
1517                    (new AMessage)->postReply(mReplyID);
1518                    break;
1519                }
1520
1521                default:
1522                    TRESPASS();
1523            }
1524            break;
1525        }
1526
1527        case kWhatInit:
1528        {
1529            sp<AReplyToken> replyID;
1530            CHECK(msg->senderAwaitsResponse(&replyID));
1531
1532            if (mState != UNINITIALIZED) {
1533                PostReplyWithError(replyID, INVALID_OPERATION);
1534                break;
1535            }
1536
1537            mReplyID = replyID;
1538            setState(INITIALIZING);
1539
1540            AString name;
1541            CHECK(msg->findString("name", &name));
1542
1543            int32_t nameIsType;
1544            int32_t encoder = false;
1545            CHECK(msg->findInt32("nameIsType", &nameIsType));
1546            if (nameIsType) {
1547                CHECK(msg->findInt32("encoder", &encoder));
1548            }
1549
1550            sp<AMessage> format = new AMessage;
1551
1552            if (nameIsType) {
1553                format->setString("mime", name.c_str());
1554                format->setInt32("encoder", encoder);
1555            } else {
1556                format->setString("componentName", name.c_str());
1557            }
1558
1559            mCodec->initiateAllocateComponent(format);
1560            break;
1561        }
1562
1563        case kWhatSetNotification:
1564        {
1565            sp<AMessage> notify;
1566            if (msg->findMessage("on-frame-rendered", &notify)) {
1567                mOnFrameRenderedNotification = notify;
1568            }
1569            break;
1570        }
1571
1572        case kWhatSetCallback:
1573        {
1574            sp<AReplyToken> replyID;
1575            CHECK(msg->senderAwaitsResponse(&replyID));
1576
1577            if (mState == UNINITIALIZED
1578                    || mState == INITIALIZING
1579                    || isExecuting()) {
1580                // callback can't be set after codec is executing,
1581                // or before it's initialized (as the callback
1582                // will be cleared when it goes to INITIALIZED)
1583                PostReplyWithError(replyID, INVALID_OPERATION);
1584                break;
1585            }
1586
1587            sp<AMessage> callback;
1588            CHECK(msg->findMessage("callback", &callback));
1589
1590            mCallback = callback;
1591
1592            if (mCallback != NULL) {
1593                ALOGI("MediaCodec will operate in async mode");
1594                mFlags |= kFlagIsAsync;
1595            } else {
1596                mFlags &= ~kFlagIsAsync;
1597            }
1598
1599            sp<AMessage> response = new AMessage;
1600            response->postReply(replyID);
1601            break;
1602        }
1603
1604        case kWhatConfigure:
1605        {
1606            sp<AReplyToken> replyID;
1607            CHECK(msg->senderAwaitsResponse(&replyID));
1608
1609            if (mState != INITIALIZED) {
1610                PostReplyWithError(replyID, INVALID_OPERATION);
1611                break;
1612            }
1613
1614            sp<RefBase> obj;
1615            CHECK(msg->findObject("surface", &obj));
1616
1617            sp<AMessage> format;
1618            CHECK(msg->findMessage("format", &format));
1619
1620            int32_t push;
1621            if (msg->findInt32("push-blank-buffers-on-shutdown", &push) && push != 0) {
1622                mFlags |= kFlagPushBlankBuffersOnShutdown;
1623            }
1624
1625            if (obj != NULL) {
1626                format->setObject("native-window", obj);
1627                status_t err = handleSetSurface(static_cast<Surface *>(obj.get()));
1628                if (err != OK) {
1629                    PostReplyWithError(replyID, err);
1630                    break;
1631                }
1632            } else {
1633                handleSetSurface(NULL);
1634            }
1635
1636            mReplyID = replyID;
1637            setState(CONFIGURING);
1638
1639            void *crypto;
1640            if (!msg->findPointer("crypto", &crypto)) {
1641                crypto = NULL;
1642            }
1643
1644            mCrypto = static_cast<ICrypto *>(crypto);
1645
1646            uint32_t flags;
1647            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1648
1649            if (flags & CONFIGURE_FLAG_ENCODE) {
1650                format->setInt32("encoder", true);
1651                mFlags |= kFlagIsEncoder;
1652            }
1653
1654            extractCSD(format);
1655
1656            mCodec->initiateConfigureComponent(format);
1657            break;
1658        }
1659
1660        case kWhatSetSurface:
1661        {
1662            sp<AReplyToken> replyID;
1663            CHECK(msg->senderAwaitsResponse(&replyID));
1664
1665            status_t err = OK;
1666            sp<Surface> surface;
1667
1668            switch (mState) {
1669                case CONFIGURED:
1670                case STARTED:
1671                case FLUSHED:
1672                {
1673                    sp<RefBase> obj;
1674                    (void)msg->findObject("surface", &obj);
1675                    sp<Surface> surface = static_cast<Surface *>(obj.get());
1676                    if (mSurface == NULL) {
1677                        // do not support setting surface if it was not set
1678                        err = INVALID_OPERATION;
1679                    } else if (obj == NULL) {
1680                        // do not support unsetting surface
1681                        err = BAD_VALUE;
1682                    } else {
1683                        err = connectToSurface(surface);
1684                        if (err == BAD_VALUE) {
1685                            // assuming reconnecting to same surface
1686                            // TODO: check if it is the same surface
1687                            err = OK;
1688                        } else {
1689                            if (err == OK) {
1690                                if (mFlags & kFlagUsesSoftwareRenderer) {
1691                                    if (mSoftRenderer != NULL
1692                                            && (mFlags & kFlagPushBlankBuffersOnShutdown)) {
1693                                        pushBlankBuffersToNativeWindow(mSurface.get());
1694                                    }
1695                                    mSoftRenderer = new SoftwareRenderer(surface);
1696                                    // TODO: check if this was successful
1697                                } else {
1698                                    err = mCodec->setSurface(surface);
1699                                }
1700                            }
1701                            if (err == OK) {
1702                                (void)disconnectFromSurface();
1703                                mSurface = surface;
1704                            }
1705                        }
1706                    }
1707                    break;
1708                }
1709
1710                default:
1711                    err = INVALID_OPERATION;
1712                    break;
1713            }
1714
1715            PostReplyWithError(replyID, err);
1716            break;
1717        }
1718
1719        case kWhatCreateInputSurface:
1720        case kWhatSetInputSurface:
1721        {
1722            sp<AReplyToken> replyID;
1723            CHECK(msg->senderAwaitsResponse(&replyID));
1724
1725            // Must be configured, but can't have been started yet.
1726            if (mState != CONFIGURED) {
1727                PostReplyWithError(replyID, INVALID_OPERATION);
1728                break;
1729            }
1730
1731            mReplyID = replyID;
1732            if (msg->what() == kWhatCreateInputSurface) {
1733                mCodec->initiateCreateInputSurface();
1734            } else {
1735                sp<RefBase> obj;
1736                CHECK(msg->findObject("input-surface", &obj));
1737
1738                mCodec->initiateSetInputSurface(
1739                        static_cast<PersistentSurface *>(obj.get()));
1740            }
1741            break;
1742        }
1743        case kWhatStart:
1744        {
1745            sp<AReplyToken> replyID;
1746            CHECK(msg->senderAwaitsResponse(&replyID));
1747
1748            if (mState == FLUSHED) {
1749                setState(STARTED);
1750                if (mHavePendingInputBuffers) {
1751                    onInputBufferAvailable();
1752                    mHavePendingInputBuffers = false;
1753                }
1754                mCodec->signalResume();
1755                PostReplyWithError(replyID, OK);
1756                break;
1757            } else if (mState != CONFIGURED) {
1758                PostReplyWithError(replyID, INVALID_OPERATION);
1759                break;
1760            }
1761
1762            mReplyID = replyID;
1763            setState(STARTING);
1764
1765            mCodec->initiateStart();
1766            break;
1767        }
1768
1769        case kWhatStop:
1770        case kWhatRelease:
1771        {
1772            State targetState =
1773                (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED;
1774
1775            sp<AReplyToken> replyID;
1776            CHECK(msg->senderAwaitsResponse(&replyID));
1777
1778            // already stopped/released
1779            if (mState == UNINITIALIZED && mReleasedByResourceManager) {
1780                sp<AMessage> response = new AMessage;
1781                response->setInt32("err", OK);
1782                response->postReply(replyID);
1783                break;
1784            }
1785
1786            int32_t reclaimed = 0;
1787            msg->findInt32("reclaimed", &reclaimed);
1788            if (reclaimed) {
1789                mReleasedByResourceManager = true;
1790            }
1791
1792            if (!((mFlags & kFlagIsComponentAllocated) && targetState == UNINITIALIZED) // See 1
1793                    && mState != INITIALIZED
1794                    && mState != CONFIGURED && !isExecuting()) {
1795                // 1) Permit release to shut down the component if allocated.
1796                //
1797                // 2) We may be in "UNINITIALIZED" state already and
1798                // also shutdown the encoder/decoder without the
1799                // client being aware of this if media server died while
1800                // we were being stopped. The client would assume that
1801                // after stop() returned, it would be safe to call release()
1802                // and it should be in this case, no harm to allow a release()
1803                // if we're already uninitialized.
1804                sp<AMessage> response = new AMessage;
1805                // TODO: we shouldn't throw an exception for stop/release. Change this to wait until
1806                // the previous stop/release completes and then reply with OK.
1807                status_t err = mState == targetState ? OK : INVALID_OPERATION;
1808                response->setInt32("err", err);
1809                if (err == OK && targetState == UNINITIALIZED) {
1810                    mComponentName.clear();
1811                }
1812                response->postReply(replyID);
1813                break;
1814            }
1815
1816            if (mFlags & kFlagSawMediaServerDie) {
1817                // It's dead, Jim. Don't expect initiateShutdown to yield
1818                // any useful results now...
1819                setState(UNINITIALIZED);
1820                if (targetState == UNINITIALIZED) {
1821                    mComponentName.clear();
1822                }
1823                (new AMessage)->postReply(replyID);
1824                break;
1825            }
1826
1827            mReplyID = replyID;
1828            setState(msg->what() == kWhatStop ? STOPPING : RELEASING);
1829
1830            mCodec->initiateShutdown(
1831                    msg->what() == kWhatStop /* keepComponentAllocated */);
1832
1833            returnBuffersToCodec();
1834
1835            if (mSoftRenderer != NULL && (mFlags & kFlagPushBlankBuffersOnShutdown)) {
1836                pushBlankBuffersToNativeWindow(mSurface.get());
1837            }
1838            break;
1839        }
1840
1841        case kWhatDequeueInputBuffer:
1842        {
1843            sp<AReplyToken> replyID;
1844            CHECK(msg->senderAwaitsResponse(&replyID));
1845
1846            if (mFlags & kFlagIsAsync) {
1847                ALOGE("dequeueOutputBuffer can't be used in async mode");
1848                PostReplyWithError(replyID, INVALID_OPERATION);
1849                break;
1850            }
1851
1852            if (mHaveInputSurface) {
1853                ALOGE("dequeueInputBuffer can't be used with input surface");
1854                PostReplyWithError(replyID, INVALID_OPERATION);
1855                break;
1856            }
1857
1858            if (handleDequeueInputBuffer(replyID, true /* new request */)) {
1859                break;
1860            }
1861
1862            int64_t timeoutUs;
1863            CHECK(msg->findInt64("timeoutUs", &timeoutUs));
1864
1865            if (timeoutUs == 0ll) {
1866                PostReplyWithError(replyID, -EAGAIN);
1867                break;
1868            }
1869
1870            mFlags |= kFlagDequeueInputPending;
1871            mDequeueInputReplyID = replyID;
1872
1873            if (timeoutUs > 0ll) {
1874                sp<AMessage> timeoutMsg =
1875                    new AMessage(kWhatDequeueInputTimedOut, this);
1876                timeoutMsg->setInt32(
1877                        "generation", ++mDequeueInputTimeoutGeneration);
1878                timeoutMsg->post(timeoutUs);
1879            }
1880            break;
1881        }
1882
1883        case kWhatDequeueInputTimedOut:
1884        {
1885            int32_t generation;
1886            CHECK(msg->findInt32("generation", &generation));
1887
1888            if (generation != mDequeueInputTimeoutGeneration) {
1889                // Obsolete
1890                break;
1891            }
1892
1893            CHECK(mFlags & kFlagDequeueInputPending);
1894
1895            PostReplyWithError(mDequeueInputReplyID, -EAGAIN);
1896
1897            mFlags &= ~kFlagDequeueInputPending;
1898            mDequeueInputReplyID = 0;
1899            break;
1900        }
1901
1902        case kWhatQueueInputBuffer:
1903        {
1904            sp<AReplyToken> replyID;
1905            CHECK(msg->senderAwaitsResponse(&replyID));
1906
1907            if (!isExecuting()) {
1908                PostReplyWithError(replyID, INVALID_OPERATION);
1909                break;
1910            } else if (mFlags & kFlagStickyError) {
1911                PostReplyWithError(replyID, getStickyError());
1912                break;
1913            }
1914
1915            status_t err = onQueueInputBuffer(msg);
1916
1917            PostReplyWithError(replyID, err);
1918            break;
1919        }
1920
1921        case kWhatDequeueOutputBuffer:
1922        {
1923            sp<AReplyToken> replyID;
1924            CHECK(msg->senderAwaitsResponse(&replyID));
1925
1926            if (mFlags & kFlagIsAsync) {
1927                ALOGE("dequeueOutputBuffer can't be used in async mode");
1928                PostReplyWithError(replyID, INVALID_OPERATION);
1929                break;
1930            }
1931
1932            if (handleDequeueOutputBuffer(replyID, true /* new request */)) {
1933                break;
1934            }
1935
1936            int64_t timeoutUs;
1937            CHECK(msg->findInt64("timeoutUs", &timeoutUs));
1938
1939            if (timeoutUs == 0ll) {
1940                PostReplyWithError(replyID, -EAGAIN);
1941                break;
1942            }
1943
1944            mFlags |= kFlagDequeueOutputPending;
1945            mDequeueOutputReplyID = replyID;
1946
1947            if (timeoutUs > 0ll) {
1948                sp<AMessage> timeoutMsg =
1949                    new AMessage(kWhatDequeueOutputTimedOut, this);
1950                timeoutMsg->setInt32(
1951                        "generation", ++mDequeueOutputTimeoutGeneration);
1952                timeoutMsg->post(timeoutUs);
1953            }
1954            break;
1955        }
1956
1957        case kWhatDequeueOutputTimedOut:
1958        {
1959            int32_t generation;
1960            CHECK(msg->findInt32("generation", &generation));
1961
1962            if (generation != mDequeueOutputTimeoutGeneration) {
1963                // Obsolete
1964                break;
1965            }
1966
1967            CHECK(mFlags & kFlagDequeueOutputPending);
1968
1969            PostReplyWithError(mDequeueOutputReplyID, -EAGAIN);
1970
1971            mFlags &= ~kFlagDequeueOutputPending;
1972            mDequeueOutputReplyID = 0;
1973            break;
1974        }
1975
1976        case kWhatReleaseOutputBuffer:
1977        {
1978            sp<AReplyToken> replyID;
1979            CHECK(msg->senderAwaitsResponse(&replyID));
1980
1981            if (!isExecuting()) {
1982                PostReplyWithError(replyID, INVALID_OPERATION);
1983                break;
1984            } else if (mFlags & kFlagStickyError) {
1985                PostReplyWithError(replyID, getStickyError());
1986                break;
1987            }
1988
1989            status_t err = onReleaseOutputBuffer(msg);
1990
1991            PostReplyWithError(replyID, err);
1992            break;
1993        }
1994
1995        case kWhatSignalEndOfInputStream:
1996        {
1997            sp<AReplyToken> replyID;
1998            CHECK(msg->senderAwaitsResponse(&replyID));
1999
2000            if (!isExecuting()) {
2001                PostReplyWithError(replyID, INVALID_OPERATION);
2002                break;
2003            } else if (mFlags & kFlagStickyError) {
2004                PostReplyWithError(replyID, getStickyError());
2005                break;
2006            }
2007
2008            mReplyID = replyID;
2009            mCodec->signalEndOfInputStream();
2010            break;
2011        }
2012
2013        case kWhatGetBuffers:
2014        {
2015            sp<AReplyToken> replyID;
2016            CHECK(msg->senderAwaitsResponse(&replyID));
2017            // Unfortunately widevine legacy source requires knowing all of the
2018            // codec input buffers, so we have to provide them even in async mode.
2019            int32_t widevine = 0;
2020            msg->findInt32("widevine", &widevine);
2021
2022            if (!isExecuting() || ((mFlags & kFlagIsAsync) && !widevine)) {
2023                PostReplyWithError(replyID, INVALID_OPERATION);
2024                break;
2025            } else if (mFlags & kFlagStickyError) {
2026                PostReplyWithError(replyID, getStickyError());
2027                break;
2028            }
2029
2030            int32_t portIndex;
2031            CHECK(msg->findInt32("portIndex", &portIndex));
2032
2033            Vector<sp<ABuffer> > *dstBuffers;
2034            CHECK(msg->findPointer("buffers", (void **)&dstBuffers));
2035
2036            dstBuffers->clear();
2037            const Vector<BufferInfo> &srcBuffers = mPortBuffers[portIndex];
2038
2039            for (size_t i = 0; i < srcBuffers.size(); ++i) {
2040                const BufferInfo &info = srcBuffers.itemAt(i);
2041
2042                dstBuffers->push_back(
2043                        (portIndex == kPortIndexInput && mCrypto != NULL)
2044                                ? info.mEncryptedData : info.mData);
2045            }
2046
2047            (new AMessage)->postReply(replyID);
2048            break;
2049        }
2050
2051        case kWhatFlush:
2052        {
2053            sp<AReplyToken> replyID;
2054            CHECK(msg->senderAwaitsResponse(&replyID));
2055
2056            if (!isExecuting()) {
2057                PostReplyWithError(replyID, INVALID_OPERATION);
2058                break;
2059            } else if (mFlags & kFlagStickyError) {
2060                PostReplyWithError(replyID, getStickyError());
2061                break;
2062            }
2063
2064            mReplyID = replyID;
2065            // TODO: skip flushing if already FLUSHED
2066            setState(FLUSHING);
2067
2068            mCodec->signalFlush();
2069            returnBuffersToCodec();
2070            break;
2071        }
2072
2073        case kWhatGetInputFormat:
2074        case kWhatGetOutputFormat:
2075        {
2076            sp<AMessage> format =
2077                (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat);
2078
2079            sp<AReplyToken> replyID;
2080            CHECK(msg->senderAwaitsResponse(&replyID));
2081
2082            if ((mState != CONFIGURED && mState != STARTING &&
2083                 mState != STARTED && mState != FLUSHING &&
2084                 mState != FLUSHED)
2085                    || format == NULL) {
2086                PostReplyWithError(replyID, INVALID_OPERATION);
2087                break;
2088            } else if (mFlags & kFlagStickyError) {
2089                PostReplyWithError(replyID, getStickyError());
2090                break;
2091            }
2092
2093            sp<AMessage> response = new AMessage;
2094            response->setMessage("format", format);
2095            response->postReply(replyID);
2096            break;
2097        }
2098
2099        case kWhatRequestIDRFrame:
2100        {
2101            mCodec->signalRequestIDRFrame();
2102            break;
2103        }
2104
2105        case kWhatRequestActivityNotification:
2106        {
2107            CHECK(mActivityNotify == NULL);
2108            CHECK(msg->findMessage("notify", &mActivityNotify));
2109
2110            postActivityNotificationIfPossible();
2111            break;
2112        }
2113
2114        case kWhatGetName:
2115        {
2116            sp<AReplyToken> replyID;
2117            CHECK(msg->senderAwaitsResponse(&replyID));
2118
2119            if (mComponentName.empty()) {
2120                PostReplyWithError(replyID, INVALID_OPERATION);
2121                break;
2122            }
2123
2124            sp<AMessage> response = new AMessage;
2125            response->setString("name", mComponentName.c_str());
2126            response->postReply(replyID);
2127            break;
2128        }
2129
2130        case kWhatSetParameters:
2131        {
2132            sp<AReplyToken> replyID;
2133            CHECK(msg->senderAwaitsResponse(&replyID));
2134
2135            sp<AMessage> params;
2136            CHECK(msg->findMessage("params", &params));
2137
2138            status_t err = onSetParameters(params);
2139
2140            PostReplyWithError(replyID, err);
2141            break;
2142        }
2143
2144        default:
2145            TRESPASS();
2146    }
2147}
2148
2149void MediaCodec::extractCSD(const sp<AMessage> &format) {
2150    mCSD.clear();
2151
2152    size_t i = 0;
2153    for (;;) {
2154        sp<ABuffer> csd;
2155        if (!format->findBuffer(AStringPrintf("csd-%u", i).c_str(), &csd)) {
2156            break;
2157        }
2158
2159        mCSD.push_back(csd);
2160        ++i;
2161    }
2162
2163    ALOGV("Found %zu pieces of codec specific data.", mCSD.size());
2164}
2165
2166status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) {
2167    CHECK(!mCSD.empty());
2168
2169    const BufferInfo *info =
2170        &mPortBuffers[kPortIndexInput].itemAt(bufferIndex);
2171
2172    sp<ABuffer> csd = *mCSD.begin();
2173    mCSD.erase(mCSD.begin());
2174
2175    const sp<ABuffer> &codecInputData =
2176        (mCrypto != NULL) ? info->mEncryptedData : info->mData;
2177
2178    if (csd->size() > codecInputData->capacity()) {
2179        return -EINVAL;
2180    }
2181
2182    memcpy(codecInputData->data(), csd->data(), csd->size());
2183
2184    AString errorDetailMsg;
2185
2186    sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
2187    msg->setSize("index", bufferIndex);
2188    msg->setSize("offset", 0);
2189    msg->setSize("size", csd->size());
2190    msg->setInt64("timeUs", 0ll);
2191    msg->setInt32("flags", BUFFER_FLAG_CODECCONFIG);
2192    msg->setPointer("errorDetailMsg", &errorDetailMsg);
2193
2194    return onQueueInputBuffer(msg);
2195}
2196
2197void MediaCodec::setState(State newState) {
2198    if (newState == INITIALIZED || newState == UNINITIALIZED) {
2199        delete mSoftRenderer;
2200        mSoftRenderer = NULL;
2201
2202        mCrypto.clear();
2203        handleSetSurface(NULL);
2204
2205        mInputFormat.clear();
2206        mOutputFormat.clear();
2207        mFlags &= ~kFlagOutputFormatChanged;
2208        mFlags &= ~kFlagOutputBuffersChanged;
2209        mFlags &= ~kFlagStickyError;
2210        mFlags &= ~kFlagIsEncoder;
2211        mFlags &= ~kFlagGatherCodecSpecificData;
2212        mFlags &= ~kFlagIsAsync;
2213        mStickyError = OK;
2214
2215        mActivityNotify.clear();
2216        mCallback.clear();
2217    }
2218
2219    if (newState == UNINITIALIZED) {
2220        // return any straggling buffers, e.g. if we got here on an error
2221        returnBuffersToCodec();
2222
2223        // The component is gone, mediaserver's probably back up already
2224        // but should definitely be back up should we try to instantiate
2225        // another component.. and the cycle continues.
2226        mFlags &= ~kFlagSawMediaServerDie;
2227    }
2228
2229    mState = newState;
2230
2231    cancelPendingDequeueOperations();
2232
2233    updateBatteryStat();
2234}
2235
2236void MediaCodec::returnBuffersToCodec() {
2237    returnBuffersToCodecOnPort(kPortIndexInput);
2238    returnBuffersToCodecOnPort(kPortIndexOutput);
2239}
2240
2241void MediaCodec::returnBuffersToCodecOnPort(int32_t portIndex) {
2242    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
2243    Mutex::Autolock al(mBufferLock);
2244
2245    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
2246
2247    for (size_t i = 0; i < buffers->size(); ++i) {
2248        BufferInfo *info = &buffers->editItemAt(i);
2249
2250        if (info->mNotify != NULL) {
2251            sp<AMessage> msg = info->mNotify;
2252            info->mNotify = NULL;
2253            info->mOwnedByClient = false;
2254
2255            if (portIndex == kPortIndexInput) {
2256                /* no error, just returning buffers */
2257                msg->setInt32("err", OK);
2258            }
2259            msg->post();
2260        }
2261    }
2262
2263    mAvailPortBuffers[portIndex].clear();
2264}
2265
2266size_t MediaCodec::updateBuffers(
2267        int32_t portIndex, const sp<AMessage> &msg) {
2268    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
2269
2270    uint32_t bufferID;
2271    CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID));
2272
2273    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
2274
2275    for (size_t i = 0; i < buffers->size(); ++i) {
2276        BufferInfo *info = &buffers->editItemAt(i);
2277
2278        if (info->mBufferID == bufferID) {
2279            CHECK(info->mNotify == NULL);
2280            CHECK(msg->findMessage("reply", &info->mNotify));
2281
2282            info->mFormat =
2283                (portIndex == kPortIndexInput) ? mInputFormat : mOutputFormat;
2284            mAvailPortBuffers[portIndex].push_back(i);
2285
2286            return i;
2287        }
2288    }
2289
2290    TRESPASS();
2291
2292    return 0;
2293}
2294
2295status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) {
2296    size_t index;
2297    size_t offset;
2298    size_t size;
2299    int64_t timeUs;
2300    uint32_t flags;
2301    CHECK(msg->findSize("index", &index));
2302    CHECK(msg->findSize("offset", &offset));
2303    CHECK(msg->findInt64("timeUs", &timeUs));
2304    CHECK(msg->findInt32("flags", (int32_t *)&flags));
2305
2306    const CryptoPlugin::SubSample *subSamples;
2307    size_t numSubSamples;
2308    const uint8_t *key;
2309    const uint8_t *iv;
2310    CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted;
2311
2312    // We allow the simpler queueInputBuffer API to be used even in
2313    // secure mode, by fabricating a single unencrypted subSample.
2314    CryptoPlugin::SubSample ss;
2315
2316    if (msg->findSize("size", &size)) {
2317        if (mCrypto != NULL) {
2318            ss.mNumBytesOfClearData = size;
2319            ss.mNumBytesOfEncryptedData = 0;
2320
2321            subSamples = &ss;
2322            numSubSamples = 1;
2323            key = NULL;
2324            iv = NULL;
2325        }
2326    } else {
2327        if (mCrypto == NULL) {
2328            return -EINVAL;
2329        }
2330
2331        CHECK(msg->findPointer("subSamples", (void **)&subSamples));
2332        CHECK(msg->findSize("numSubSamples", &numSubSamples));
2333        CHECK(msg->findPointer("key", (void **)&key));
2334        CHECK(msg->findPointer("iv", (void **)&iv));
2335
2336        int32_t tmp;
2337        CHECK(msg->findInt32("mode", &tmp));
2338
2339        mode = (CryptoPlugin::Mode)tmp;
2340
2341        size = 0;
2342        for (size_t i = 0; i < numSubSamples; ++i) {
2343            size += subSamples[i].mNumBytesOfClearData;
2344            size += subSamples[i].mNumBytesOfEncryptedData;
2345        }
2346    }
2347
2348    if (index >= mPortBuffers[kPortIndexInput].size()) {
2349        return -ERANGE;
2350    }
2351
2352    BufferInfo *info = &mPortBuffers[kPortIndexInput].editItemAt(index);
2353
2354    if (info->mNotify == NULL || !info->mOwnedByClient) {
2355        return -EACCES;
2356    }
2357
2358    if (offset + size > info->mData->capacity()) {
2359        return -EINVAL;
2360    }
2361
2362    sp<AMessage> reply = info->mNotify;
2363    info->mData->setRange(offset, size);
2364    info->mData->meta()->setInt64("timeUs", timeUs);
2365
2366    if (flags & BUFFER_FLAG_EOS) {
2367        info->mData->meta()->setInt32("eos", true);
2368    }
2369
2370    if (flags & BUFFER_FLAG_CODECCONFIG) {
2371        info->mData->meta()->setInt32("csd", true);
2372    }
2373
2374    if (mCrypto != NULL) {
2375        if (size > info->mEncryptedData->capacity()) {
2376            return -ERANGE;
2377        }
2378
2379        AString *errorDetailMsg;
2380        CHECK(msg->findPointer("errorDetailMsg", (void **)&errorDetailMsg));
2381
2382        ssize_t result = mCrypto->decrypt(
2383                (mFlags & kFlagIsSecure) != 0,
2384                key,
2385                iv,
2386                mode,
2387                info->mSharedEncryptedBuffer,
2388                offset,
2389                subSamples,
2390                numSubSamples,
2391                info->mData->base(),
2392                errorDetailMsg);
2393
2394        if (result < 0) {
2395            return result;
2396        }
2397
2398        info->mData->setRange(0, result);
2399    }
2400
2401    // synchronization boundary for getBufferAndFormat
2402    {
2403        Mutex::Autolock al(mBufferLock);
2404        info->mOwnedByClient = false;
2405    }
2406    reply->setBuffer("buffer", info->mData);
2407    reply->post();
2408
2409    info->mNotify = NULL;
2410
2411    return OK;
2412}
2413
2414//static
2415size_t MediaCodec::CreateFramesRenderedMessage(
2416        std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg) {
2417    size_t index = 0;
2418
2419    for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
2420            it != done.cend(); ++it) {
2421        if (it->getRenderTimeNs() < 0) {
2422            continue; // dropped frame from tracking
2423        }
2424        msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs());
2425        msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs());
2426        ++index;
2427    }
2428    return index;
2429}
2430
2431status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
2432    size_t index;
2433    CHECK(msg->findSize("index", &index));
2434
2435    int32_t render;
2436    if (!msg->findInt32("render", &render)) {
2437        render = 0;
2438    }
2439
2440    if (!isExecuting()) {
2441        return -EINVAL;
2442    }
2443
2444    if (index >= mPortBuffers[kPortIndexOutput].size()) {
2445        return -ERANGE;
2446    }
2447
2448    BufferInfo *info = &mPortBuffers[kPortIndexOutput].editItemAt(index);
2449
2450    if (info->mNotify == NULL || !info->mOwnedByClient) {
2451        return -EACCES;
2452    }
2453
2454    // synchronization boundary for getBufferAndFormat
2455    {
2456        Mutex::Autolock al(mBufferLock);
2457        info->mOwnedByClient = false;
2458    }
2459
2460    if (render && info->mData != NULL && info->mData->size() != 0) {
2461        info->mNotify->setInt32("render", true);
2462
2463        int64_t mediaTimeUs = -1;
2464        info->mData->meta()->findInt64("timeUs", &mediaTimeUs);
2465
2466        int64_t renderTimeNs = 0;
2467        if (!msg->findInt64("timestampNs", &renderTimeNs)) {
2468            // use media timestamp if client did not request a specific render timestamp
2469            ALOGV("using buffer PTS of %lld", (long long)mediaTimeUs);
2470            renderTimeNs = mediaTimeUs * 1000;
2471        }
2472        info->mNotify->setInt64("timestampNs", renderTimeNs);
2473
2474        if (mSoftRenderer != NULL) {
2475            std::list<FrameRenderTracker::Info> doneFrames = mSoftRenderer->render(
2476                    info->mData->data(), info->mData->size(),
2477                    mediaTimeUs, renderTimeNs, NULL, info->mFormat);
2478
2479            // if we are running, notify rendered frames
2480            if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) {
2481                sp<AMessage> notify = mOnFrameRenderedNotification->dup();
2482                sp<AMessage> data = new AMessage;
2483                if (CreateFramesRenderedMessage(doneFrames, data)) {
2484                    notify->setMessage("data", data);
2485                    notify->post();
2486                }
2487            }
2488        }
2489    }
2490
2491    info->mNotify->post();
2492    info->mNotify = NULL;
2493
2494    return OK;
2495}
2496
2497ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {
2498    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
2499
2500    List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];
2501
2502    if (availBuffers->empty()) {
2503        return -EAGAIN;
2504    }
2505
2506    size_t index = *availBuffers->begin();
2507    availBuffers->erase(availBuffers->begin());
2508
2509    BufferInfo *info = &mPortBuffers[portIndex].editItemAt(index);
2510    CHECK(!info->mOwnedByClient);
2511    {
2512        Mutex::Autolock al(mBufferLock);
2513        info->mOwnedByClient = true;
2514
2515        // set image-data
2516        if (info->mFormat != NULL) {
2517            sp<ABuffer> imageData;
2518            if (info->mFormat->findBuffer("image-data", &imageData)) {
2519                info->mData->meta()->setBuffer("image-data", imageData);
2520            }
2521            int32_t left, top, right, bottom;
2522            if (info->mFormat->findRect("crop", &left, &top, &right, &bottom)) {
2523                info->mData->meta()->setRect("crop-rect", left, top, right, bottom);
2524            }
2525        }
2526    }
2527
2528    return index;
2529}
2530
2531status_t MediaCodec::connectToSurface(const sp<Surface> &surface) {
2532    status_t err = OK;
2533    if (surface != NULL) {
2534        err = native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA);
2535        if (err == BAD_VALUE) {
2536            ALOGI("native window already connected. Assuming no change of surface");
2537            return err;
2538        } else if (err == OK) {
2539            // Require a fresh set of buffers after each connect by using a unique generation
2540            // number. Rely on the fact that max supported process id by Linux is 2^22.
2541            // PID is never 0 so we don't have to worry that we use the default generation of 0.
2542            // TODO: come up with a unique scheme if other producers also set the generation number.
2543            static uint32_t mSurfaceGeneration = 0;
2544            uint32_t generation = (getpid() << 10) | (++mSurfaceGeneration & ((1 << 10) - 1));
2545            surface->setGenerationNumber(generation);
2546            ALOGI("[%s] setting surface generation to %u", mComponentName.c_str(), generation);
2547
2548            // HACK: clear any free buffers. Remove when connect will automatically do this.
2549            // This is needed as the consumer may be holding onto stale frames that it can reattach
2550            // to this surface after disconnect/connect, and those free frames would inherit the new
2551            // generation number. Disconnecting after setting a unique generation prevents this.
2552            native_window_api_disconnect(surface.get(), NATIVE_WINDOW_API_MEDIA);
2553            err = native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA);
2554        }
2555
2556        if (err != OK) {
2557            ALOGE("native_window_api_connect returned an error: %s (%d)", strerror(-err), err);
2558        }
2559    }
2560    return err;
2561}
2562
2563status_t MediaCodec::disconnectFromSurface() {
2564    status_t err = OK;
2565    if (mSurface != NULL) {
2566        // Resetting generation is not technically needed, but there is no need to keep it either
2567        mSurface->setGenerationNumber(0);
2568        err = native_window_api_disconnect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
2569        if (err != OK) {
2570            ALOGW("native_window_api_disconnect returned an error: %s (%d)", strerror(-err), err);
2571        }
2572        // assume disconnected even on error
2573        mSurface.clear();
2574    }
2575    return err;
2576}
2577
2578status_t MediaCodec::handleSetSurface(const sp<Surface> &surface) {
2579    status_t err = OK;
2580    if (mSurface != NULL) {
2581        (void)disconnectFromSurface();
2582    }
2583    if (surface != NULL) {
2584        err = connectToSurface(surface);
2585        if (err == OK) {
2586            mSurface = surface;
2587        }
2588    }
2589    return err;
2590}
2591
2592void MediaCodec::onInputBufferAvailable() {
2593    int32_t index;
2594    while ((index = dequeuePortBuffer(kPortIndexInput)) >= 0) {
2595        sp<AMessage> msg = mCallback->dup();
2596        msg->setInt32("callbackID", CB_INPUT_AVAILABLE);
2597        msg->setInt32("index", index);
2598        msg->post();
2599    }
2600}
2601
2602void MediaCodec::onOutputBufferAvailable() {
2603    int32_t index;
2604    while ((index = dequeuePortBuffer(kPortIndexOutput)) >= 0) {
2605        const sp<ABuffer> &buffer =
2606            mPortBuffers[kPortIndexOutput].itemAt(index).mData;
2607        sp<AMessage> msg = mCallback->dup();
2608        msg->setInt32("callbackID", CB_OUTPUT_AVAILABLE);
2609        msg->setInt32("index", index);
2610        msg->setSize("offset", buffer->offset());
2611        msg->setSize("size", buffer->size());
2612
2613        int64_t timeUs;
2614        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2615
2616        msg->setInt64("timeUs", timeUs);
2617
2618        int32_t omxFlags;
2619        CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags));
2620
2621        uint32_t flags = 0;
2622        if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) {
2623            flags |= BUFFER_FLAG_SYNCFRAME;
2624        }
2625        if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) {
2626            flags |= BUFFER_FLAG_CODECCONFIG;
2627        }
2628        if (omxFlags & OMX_BUFFERFLAG_EOS) {
2629            flags |= BUFFER_FLAG_EOS;
2630        }
2631
2632        msg->setInt32("flags", flags);
2633
2634        msg->post();
2635    }
2636}
2637
2638void MediaCodec::onError(status_t err, int32_t actionCode, const char *detail) {
2639    if (mCallback != NULL) {
2640        sp<AMessage> msg = mCallback->dup();
2641        msg->setInt32("callbackID", CB_ERROR);
2642        msg->setInt32("err", err);
2643        msg->setInt32("actionCode", actionCode);
2644
2645        if (detail != NULL) {
2646            msg->setString("detail", detail);
2647        }
2648
2649        msg->post();
2650    }
2651}
2652
2653void MediaCodec::onOutputFormatChanged() {
2654    if (mCallback != NULL) {
2655        sp<AMessage> msg = mCallback->dup();
2656        msg->setInt32("callbackID", CB_OUTPUT_FORMAT_CHANGED);
2657        msg->setMessage("format", mOutputFormat);
2658        msg->post();
2659    }
2660}
2661
2662
2663void MediaCodec::postActivityNotificationIfPossible() {
2664    if (mActivityNotify == NULL) {
2665        return;
2666    }
2667
2668    bool isErrorOrOutputChanged =
2669            (mFlags & (kFlagStickyError
2670                    | kFlagOutputBuffersChanged
2671                    | kFlagOutputFormatChanged));
2672
2673    if (isErrorOrOutputChanged
2674            || !mAvailPortBuffers[kPortIndexInput].empty()
2675            || !mAvailPortBuffers[kPortIndexOutput].empty()) {
2676        mActivityNotify->setInt32("input-buffers",
2677                mAvailPortBuffers[kPortIndexInput].size());
2678
2679        if (isErrorOrOutputChanged) {
2680            // we want consumer to dequeue as many times as it can
2681            mActivityNotify->setInt32("output-buffers", INT32_MAX);
2682        } else {
2683            mActivityNotify->setInt32("output-buffers",
2684                    mAvailPortBuffers[kPortIndexOutput].size());
2685        }
2686        mActivityNotify->post();
2687        mActivityNotify.clear();
2688    }
2689}
2690
2691status_t MediaCodec::setParameters(const sp<AMessage> &params) {
2692    sp<AMessage> msg = new AMessage(kWhatSetParameters, this);
2693    msg->setMessage("params", params);
2694
2695    sp<AMessage> response;
2696    return PostAndAwaitResponse(msg, &response);
2697}
2698
2699status_t MediaCodec::onSetParameters(const sp<AMessage> &params) {
2700    mCodec->signalSetParameters(params);
2701
2702    return OK;
2703}
2704
2705status_t MediaCodec::amendOutputFormatWithCodecSpecificData(
2706        const sp<ABuffer> &buffer) {
2707    AString mime;
2708    CHECK(mOutputFormat->findString("mime", &mime));
2709
2710    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) {
2711        // Codec specific data should be SPS and PPS in a single buffer,
2712        // each prefixed by a startcode (0x00 0x00 0x00 0x01).
2713        // We separate the two and put them into the output format
2714        // under the keys "csd-0" and "csd-1".
2715
2716        unsigned csdIndex = 0;
2717
2718        const uint8_t *data = buffer->data();
2719        size_t size = buffer->size();
2720
2721        const uint8_t *nalStart;
2722        size_t nalSize;
2723        while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) {
2724            sp<ABuffer> csd = new ABuffer(nalSize + 4);
2725            memcpy(csd->data(), "\x00\x00\x00\x01", 4);
2726            memcpy(csd->data() + 4, nalStart, nalSize);
2727
2728            mOutputFormat->setBuffer(
2729                    AStringPrintf("csd-%u", csdIndex).c_str(), csd);
2730
2731            ++csdIndex;
2732        }
2733
2734        if (csdIndex != 2) {
2735            return ERROR_MALFORMED;
2736        }
2737    } else {
2738        // For everything else we just stash the codec specific data into
2739        // the output format as a single piece of csd under "csd-0".
2740        mOutputFormat->setBuffer("csd-0", buffer);
2741    }
2742
2743    return OK;
2744}
2745
2746void MediaCodec::updateBatteryStat() {
2747    if (mState == CONFIGURED && !mBatteryStatNotified) {
2748        BatteryNotifier& notifier(BatteryNotifier::getInstance());
2749
2750        if (mIsVideo) {
2751            notifier.noteStartVideo();
2752        } else {
2753            notifier.noteStartAudio();
2754        }
2755
2756        mBatteryStatNotified = true;
2757    } else if (mState == UNINITIALIZED && mBatteryStatNotified) {
2758        BatteryNotifier& notifier(BatteryNotifier::getInstance());
2759
2760        if (mIsVideo) {
2761            notifier.noteStopVideo();
2762        } else {
2763            notifier.noteStopAudio();
2764        }
2765
2766        mBatteryStatNotified = false;
2767    }
2768}
2769
2770}  // namespace android
2771