MediaCodec.cpp revision c6044a135324979e62cc59db9dbde8b11c4bb74b 
1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "MediaCodec"
19#include <inttypes.h>
20
21#include "include/avc_utils.h"
22#include "include/SoftwareRenderer.h"
23
24#include <binder/IMemory.h>
25#include <binder/IPCThreadState.h>
26#include <binder/IServiceManager.h>
27#include <binder/MemoryDealer.h>
28#include <gui/BufferQueue.h>
29#include <gui/Surface.h>
30#include <media/ICrypto.h>
31#include <media/IOMX.h>
32#include <media/IResourceManagerService.h>
33#include <media/stagefright/foundation/ABuffer.h>
34#include <media/stagefright/foundation/ADebug.h>
35#include <media/stagefright/foundation/AMessage.h>
36#include <media/stagefright/foundation/AString.h>
37#include <media/stagefright/foundation/hexdump.h>
38#include <media/stagefright/ACodec.h>
39#include <media/stagefright/BufferProducerWrapper.h>
40#include <media/stagefright/MediaCodec.h>
41#include <media/stagefright/MediaCodecList.h>
42#include <media/stagefright/MediaDefs.h>
43#include <media/stagefright/MediaErrors.h>
44#include <media/stagefright/MediaFilter.h>
45#include <media/stagefright/MetaData.h>
46#include <media/stagefright/OMXClient.h>
47#include <media/stagefright/OMXCodec.h>
48#include <media/stagefright/PersistentSurface.h>
49#include <media/stagefright/SurfaceUtils.h>
50#include <mediautils/BatteryNotifier.h>
51#include <private/android_filesystem_config.h>
52#include <utils/Log.h>
53#include <utils/Singleton.h>
54
55namespace android {
56
57static inline int getCallingPid() {
58    return IPCThreadState::self()->getCallingPid();
59}
60
61static int64_t getId(sp<IResourceManagerClient> client) {
62    return (int64_t) client.get();
63}
64
65static bool isResourceError(status_t err) {
66    return (err == NO_MEMORY);
67}
68
69static const int kMaxRetry = 2;
70
71struct ResourceManagerClient : public BnResourceManagerClient {
72    ResourceManagerClient(MediaCodec* codec) : mMediaCodec(codec) {}
73
74    virtual bool reclaimResource() {
75        sp<MediaCodec> codec = mMediaCodec.promote();
76        if (codec == NULL) {
77            // codec is already gone.
78            return true;
79        }
80        status_t err = codec->reclaim();
81        if (err != OK) {
82            ALOGW("ResourceManagerClient failed to release codec with err %d", err);
83        }
84        return (err == OK);
85    }
86
87    virtual String8 getName() {
88        String8 ret;
89        sp<MediaCodec> codec = mMediaCodec.promote();
90        if (codec == NULL) {
91            // codec is already gone.
92            return ret;
93        }
94
95        AString name;
96        if (codec->getName(&name) == OK) {
97            ret.setTo(name.c_str());
98        }
99        return ret;
100    }
101
102protected:
103    virtual ~ResourceManagerClient() {}
104
105private:
106    wp<MediaCodec> mMediaCodec;
107
108    DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient);
109};
110
111MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy() {
112}
113
114MediaCodec::ResourceManagerServiceProxy::~ResourceManagerServiceProxy() {
115    if (mService != NULL) {
116        IInterface::asBinder(mService)->unlinkToDeath(this);
117    }
118}
119
120void MediaCodec::ResourceManagerServiceProxy::init() {
121    sp<IServiceManager> sm = defaultServiceManager();
122    sp<IBinder> binder = sm->getService(String16("media.resource_manager"));
123    mService = interface_cast<IResourceManagerService>(binder);
124    if (mService == NULL) {
125        ALOGE("Failed to get ResourceManagerService");
126        return;
127    }
128    IInterface::asBinder(mService)->linkToDeath(this);
129}
130
131void MediaCodec::ResourceManagerServiceProxy::binderDied(const wp<IBinder>& /*who*/) {
132    ALOGW("ResourceManagerService died.");
133    Mutex::Autolock _l(mLock);
134    mService.clear();
135}
136
137void MediaCodec::ResourceManagerServiceProxy::addResource(
138        int pid,
139        int64_t clientId,
140        const sp<IResourceManagerClient> client,
141        const Vector<MediaResource> &resources) {
142    Mutex::Autolock _l(mLock);
143    if (mService == NULL) {
144        return;
145    }
146    mService->addResource(pid, clientId, client, resources);
147}
148
149void MediaCodec::ResourceManagerServiceProxy::removeResource(int64_t clientId) {
150    Mutex::Autolock _l(mLock);
151    if (mService == NULL) {
152        return;
153    }
154    mService->removeResource(clientId);
155}
156
157bool MediaCodec::ResourceManagerServiceProxy::reclaimResource(
158        int callingPid, const Vector<MediaResource> &resources) {
159    Mutex::Autolock _l(mLock);
160    if (mService == NULL) {
161        return false;
162    }
163    return mService->reclaimResource(callingPid, resources);
164}
165
166// static
167sp<MediaCodec> MediaCodec::CreateByType(
168        const sp<ALooper> &looper, const char *mime, bool encoder, status_t *err) {
169    sp<MediaCodec> codec = new MediaCodec(looper);
170
171    const status_t ret = codec->init(mime, true /* nameIsType */, encoder);
172    if (err != NULL) {
173        *err = ret;
174    }
175    return ret == OK ? codec : NULL; // NULL deallocates codec.
176}
177
178// static
179sp<MediaCodec> MediaCodec::CreateByComponentName(
180        const sp<ALooper> &looper, const char *name, status_t *err) {
181    sp<MediaCodec> codec = new MediaCodec(looper);
182
183    const status_t ret = codec->init(name, false /* nameIsType */, false /* encoder */);
184    if (err != NULL) {
185        *err = ret;
186    }
187    return ret == OK ? codec : NULL; // NULL deallocates codec.
188}
189
190// static
191sp<PersistentSurface> MediaCodec::CreatePersistentInputSurface() {
192    OMXClient client;
193    CHECK_EQ(client.connect(), (status_t)OK);
194    sp<IOMX> omx = client.interface();
195
196    const sp<IMediaCodecList> mediaCodecList = MediaCodecList::getInstance();
197    if (mediaCodecList == NULL) {
198        ALOGE("Failed to obtain MediaCodecList!");
199        return NULL; // if called from Java should raise IOException
200    }
201
202    AString tmp;
203    sp<AMessage> globalSettings = mediaCodecList->getGlobalSettings();
204    if (globalSettings == NULL || !globalSettings->findString(
205            kMaxEncoderInputBuffers, &tmp)) {
206        ALOGE("Failed to get encoder input buffer count!");
207        return NULL;
208    }
209
210    int32_t bufferCount = strtol(tmp.c_str(), NULL, 10);
211    if (bufferCount <= 0
212            || bufferCount > BufferQueue::MAX_MAX_ACQUIRED_BUFFERS) {
213        ALOGE("Encoder input buffer count is invalid!");
214        return NULL;
215    }
216
217    sp<IGraphicBufferProducer> bufferProducer;
218    sp<IGraphicBufferConsumer> bufferConsumer;
219
220    status_t err = omx->createPersistentInputSurface(
221            &bufferProducer, &bufferConsumer);
222
223    if (err != OK) {
224        ALOGE("Failed to create persistent input surface.");
225        return NULL;
226    }
227
228    err = bufferConsumer->setMaxAcquiredBufferCount(bufferCount);
229
230    if (err != NO_ERROR) {
231        ALOGE("Unable to set BQ max acquired buffer count to %u: %d",
232                bufferCount, err);
233        return NULL;
234    }
235
236    return new PersistentSurface(bufferProducer, bufferConsumer);
237}
238
239MediaCodec::MediaCodec(const sp<ALooper> &looper)
240    : mState(UNINITIALIZED),
241      mReleasedByResourceManager(false),
242      mLooper(looper),
243      mCodec(NULL),
244      mReplyID(0),
245      mFlags(0),
246      mStickyError(OK),
247      mSoftRenderer(NULL),
248      mResourceManagerClient(new ResourceManagerClient(this)),
249      mResourceManagerService(new ResourceManagerServiceProxy()),
250      mBatteryStatNotified(false),
251      mIsVideo(false),
252      mVideoWidth(0),
253      mVideoHeight(0),
254      mDequeueInputTimeoutGeneration(0),
255      mDequeueInputReplyID(0),
256      mDequeueOutputTimeoutGeneration(0),
257      mDequeueOutputReplyID(0),
258      mHaveInputSurface(false),
259      mHavePendingInputBuffers(false) {
260}
261
262MediaCodec::~MediaCodec() {
263    CHECK_EQ(mState, UNINITIALIZED);
264    mResourceManagerService->removeResource(getId(mResourceManagerClient));
265}
266
267// static
268status_t MediaCodec::PostAndAwaitResponse(
269        const sp<AMessage> &msg, sp<AMessage> *response) {
270    status_t err = msg->postAndAwaitResponse(response);
271
272    if (err != OK) {
273        return err;
274    }
275
276    if (!(*response)->findInt32("err", &err)) {
277        err = OK;
278    }
279
280    return err;
281}
282
283void MediaCodec::PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err) {
284    int32_t finalErr = err;
285    if (mReleasedByResourceManager) {
286        // override the err code if MediaCodec has been released by ResourceManager.
287        finalErr = DEAD_OBJECT;
288    }
289
290    sp<AMessage> response = new AMessage;
291    response->setInt32("err", finalErr);
292    response->postReply(replyID);
293}
294
295status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) {
296    mResourceManagerService->init();
297
298    // save init parameters for reset
299    mInitName = name;
300    mInitNameIsType = nameIsType;
301    mInitIsEncoder = encoder;
302
303    // Current video decoders do not return from OMX_FillThisBuffer
304    // quickly, violating the OpenMAX specs, until that is remedied
305    // we need to invest in an extra looper to free the main event
306    // queue.
307
308    if (nameIsType || !strncasecmp(name.c_str(), "omx.", 4)) {
309        mCodec = new ACodec;
310    } else if (!nameIsType
311            && !strncasecmp(name.c_str(), "android.filter.", 15)) {
312        mCodec = new MediaFilter;
313    } else {
314        return NAME_NOT_FOUND;
315    }
316
317    bool secureCodec = false;
318    if (nameIsType && !strncasecmp(name.c_str(), "video/", 6)) {
319        mIsVideo = true;
320    } else {
321        AString tmp = name;
322        if (tmp.endsWith(".secure")) {
323            secureCodec = true;
324            tmp.erase(tmp.size() - 7, 7);
325        }
326        const sp<IMediaCodecList> mcl = MediaCodecList::getInstance();
327        if (mcl == NULL) {
328            mCodec = NULL;  // remove the codec.
329            return NO_INIT; // if called from Java should raise IOException
330        }
331        ssize_t codecIdx = mcl->findCodecByName(tmp.c_str());
332        if (codecIdx >= 0) {
333            const sp<MediaCodecInfo> info = mcl->getCodecInfo(codecIdx);
334            Vector<AString> mimes;
335            info->getSupportedMimes(&mimes);
336            for (size_t i = 0; i < mimes.size(); i++) {
337                if (mimes[i].startsWith("video/")) {
338                    mIsVideo = true;
339                    break;
340                }
341            }
342        }
343    }
344
345    if (mIsVideo) {
346        // video codec needs dedicated looper
347        if (mCodecLooper == NULL) {
348            mCodecLooper = new ALooper;
349            mCodecLooper->setName("CodecLooper");
350            mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
351        }
352
353        mCodecLooper->registerHandler(mCodec);
354    } else {
355        mLooper->registerHandler(mCodec);
356    }
357
358    mLooper->registerHandler(this);
359
360    mCodec->setNotificationMessage(new AMessage(kWhatCodecNotify, this));
361
362    sp<AMessage> msg = new AMessage(kWhatInit, this);
363    msg->setString("name", name);
364    msg->setInt32("nameIsType", nameIsType);
365
366    if (nameIsType) {
367        msg->setInt32("encoder", encoder);
368    }
369
370    status_t err;
371    Vector<MediaResource> resources;
372    const char *type = secureCodec ? kResourceSecureCodec : kResourceNonSecureCodec;
373    const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec;
374    resources.push_back(MediaResource(String8(type), String8(subtype), 1));
375    for (int i = 0; i <= kMaxRetry; ++i) {
376        if (i > 0) {
377            // Don't try to reclaim resource for the first time.
378            if (!mResourceManagerService->reclaimResource(getCallingPid(), resources)) {
379                break;
380            }
381        }
382
383        sp<AMessage> response;
384        err = PostAndAwaitResponse(msg, &response);
385        if (!isResourceError(err)) {
386            break;
387        }
388    }
389    return err;
390}
391
392status_t MediaCodec::setCallback(const sp<AMessage> &callback) {
393    sp<AMessage> msg = new AMessage(kWhatSetCallback, this);
394    msg->setMessage("callback", callback);
395
396    sp<AMessage> response;
397    return PostAndAwaitResponse(msg, &response);
398}
399
400status_t MediaCodec::setOnFrameRenderedNotification(const sp<AMessage> &notify) {
401    sp<AMessage> msg = new AMessage(kWhatSetNotification, this);
402    msg->setMessage("on-frame-rendered", notify);
403    return msg->post();
404}
405
406status_t MediaCodec::configure(
407        const sp<AMessage> &format,
408        const sp<Surface> &surface,
409        const sp<ICrypto> &crypto,
410        uint32_t flags) {
411    sp<AMessage> msg = new AMessage(kWhatConfigure, this);
412
413    if (mIsVideo) {
414        format->findInt32("width", &mVideoWidth);
415        format->findInt32("height", &mVideoHeight);
416    }
417
418    msg->setMessage("format", format);
419    msg->setInt32("flags", flags);
420    msg->setObject("surface", surface);
421
422    if (crypto != NULL) {
423        msg->setPointer("crypto", crypto.get());
424    }
425
426    // save msg for reset
427    mConfigureMsg = msg;
428
429    status_t err;
430    Vector<MediaResource> resources;
431    const char *type = (mFlags & kFlagIsSecure) ?
432            kResourceSecureCodec : kResourceNonSecureCodec;
433    const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec;
434    resources.push_back(MediaResource(String8(type), String8(subtype), 1));
435    // Don't know the buffer size at this point, but it's fine to use 1 because
436    // the reclaimResource call doesn't consider the requester's buffer size for now.
437    resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1));
438    for (int i = 0; i <= kMaxRetry; ++i) {
439        if (i > 0) {
440            // Don't try to reclaim resource for the first time.
441            if (!mResourceManagerService->reclaimResource(getCallingPid(), resources)) {
442                break;
443            }
444        }
445
446        sp<AMessage> response;
447        err = PostAndAwaitResponse(msg, &response);
448        if (err != OK && err != INVALID_OPERATION) {
449            // MediaCodec now set state to UNINITIALIZED upon any fatal error.
450            // To maintain backward-compatibility, do a reset() to put codec
451            // back into INITIALIZED state.
452            // But don't reset if the err is INVALID_OPERATION, which means
453            // the configure failure is due to wrong state.
454
455            ALOGE("configure failed with err 0x%08x, resetting...", err);
456            reset();
457        }
458        if (!isResourceError(err)) {
459            break;
460        }
461    }
462    return err;
463}
464
465status_t MediaCodec::setInputSurface(
466        const sp<PersistentSurface> &surface) {
467    sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this);
468    msg->setObject("input-surface", surface.get());
469
470    sp<AMessage> response;
471    return PostAndAwaitResponse(msg, &response);
472}
473
474status_t MediaCodec::setSurface(const sp<Surface> &surface) {
475    sp<AMessage> msg = new AMessage(kWhatSetSurface, this);
476    msg->setObject("surface", surface);
477
478    sp<AMessage> response;
479    return PostAndAwaitResponse(msg, &response);
480}
481
482status_t MediaCodec::createInputSurface(
483        sp<IGraphicBufferProducer>* bufferProducer) {
484    sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, this);
485
486    sp<AMessage> response;
487    status_t err = PostAndAwaitResponse(msg, &response);
488    if (err == NO_ERROR) {
489        // unwrap the sp<IGraphicBufferProducer>
490        sp<RefBase> obj;
491        bool found = response->findObject("input-surface", &obj);
492        CHECK(found);
493        sp<BufferProducerWrapper> wrapper(
494                static_cast<BufferProducerWrapper*>(obj.get()));
495        *bufferProducer = wrapper->getBufferProducer();
496    } else {
497        ALOGW("createInputSurface failed, err=%d", err);
498    }
499    return err;
500}
501
502uint64_t MediaCodec::getGraphicBufferSize() {
503    if (!mIsVideo) {
504        return 0;
505    }
506
507    uint64_t size = 0;
508    size_t portNum = sizeof(mPortBuffers) / sizeof((mPortBuffers)[0]);
509    for (size_t i = 0; i < portNum; ++i) {
510        // TODO: this is just an estimation, we should get the real buffer size from ACodec.
511        size += mPortBuffers[i].size() * mVideoWidth * mVideoHeight * 3 / 2;
512    }
513    return size;
514}
515
516void MediaCodec::addResource(const String8 &type, const String8 &subtype, uint64_t value) {
517    Vector<MediaResource> resources;
518    resources.push_back(MediaResource(type, subtype, value));
519    mResourceManagerService->addResource(
520            getCallingPid(), getId(mResourceManagerClient), mResourceManagerClient, resources);
521}
522
523status_t MediaCodec::start() {
524    sp<AMessage> msg = new AMessage(kWhatStart, this);
525
526    status_t err;
527    Vector<MediaResource> resources;
528    const char *type = (mFlags & kFlagIsSecure) ?
529            kResourceSecureCodec : kResourceNonSecureCodec;
530    const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec;
531    resources.push_back(MediaResource(String8(type), String8(subtype), 1));
532    // Don't know the buffer size at this point, but it's fine to use 1 because
533    // the reclaimResource call doesn't consider the requester's buffer size for now.
534    resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1));
535    for (int i = 0; i <= kMaxRetry; ++i) {
536        if (i > 0) {
537            // Don't try to reclaim resource for the first time.
538            if (!mResourceManagerService->reclaimResource(getCallingPid(), resources)) {
539                break;
540            }
541            // Recover codec from previous error before retry start.
542            err = reset();
543            if (err != OK) {
544                ALOGE("retrying start: failed to reset codec");
545                break;
546            }
547            sp<AMessage> response;
548            err = PostAndAwaitResponse(mConfigureMsg, &response);
549            if (err != OK) {
550                ALOGE("retrying start: failed to configure codec");
551                break;
552            }
553        }
554
555        sp<AMessage> response;
556        err = PostAndAwaitResponse(msg, &response);
557        if (!isResourceError(err)) {
558            break;
559        }
560    }
561    return err;
562}
563
564status_t MediaCodec::stop() {
565    sp<AMessage> msg = new AMessage(kWhatStop, this);
566
567    sp<AMessage> response;
568    return PostAndAwaitResponse(msg, &response);
569}
570
571status_t MediaCodec::reclaim() {
572    sp<AMessage> msg = new AMessage(kWhatRelease, this);
573    msg->setInt32("reclaimed", 1);
574
575    sp<AMessage> response;
576    return PostAndAwaitResponse(msg, &response);
577}
578
579status_t MediaCodec::release() {
580    sp<AMessage> msg = new AMessage(kWhatRelease, this);
581
582    sp<AMessage> response;
583    return PostAndAwaitResponse(msg, &response);
584}
585
586status_t MediaCodec::reset() {
587    /* When external-facing MediaCodec object is created,
588       it is already initialized.  Thus, reset is essentially
589       release() followed by init(), plus clearing the state */
590
591    status_t err = release();
592
593    // unregister handlers
594    if (mCodec != NULL) {
595        if (mCodecLooper != NULL) {
596            mCodecLooper->unregisterHandler(mCodec->id());
597        } else {
598            mLooper->unregisterHandler(mCodec->id());
599        }
600        mCodec = NULL;
601    }
602    mLooper->unregisterHandler(id());
603
604    mFlags = 0;    // clear all flags
605    mStickyError = OK;
606
607    // reset state not reset by setState(UNINITIALIZED)
608    mReplyID = 0;
609    mDequeueInputReplyID = 0;
610    mDequeueOutputReplyID = 0;
611    mDequeueInputTimeoutGeneration = 0;
612    mDequeueOutputTimeoutGeneration = 0;
613    mHaveInputSurface = false;
614
615    if (err == OK) {
616        err = init(mInitName, mInitNameIsType, mInitIsEncoder);
617    }
618    return err;
619}
620
621status_t MediaCodec::queueInputBuffer(
622        size_t index,
623        size_t offset,
624        size_t size,
625        int64_t presentationTimeUs,
626        uint32_t flags,
627        AString *errorDetailMsg) {
628    if (errorDetailMsg != NULL) {
629        errorDetailMsg->clear();
630    }
631
632    sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
633    msg->setSize("index", index);
634    msg->setSize("offset", offset);
635    msg->setSize("size", size);
636    msg->setInt64("timeUs", presentationTimeUs);
637    msg->setInt32("flags", flags);
638    msg->setPointer("errorDetailMsg", errorDetailMsg);
639
640    sp<AMessage> response;
641    return PostAndAwaitResponse(msg, &response);
642}
643
644status_t MediaCodec::queueSecureInputBuffer(
645        size_t index,
646        size_t offset,
647        const CryptoPlugin::SubSample *subSamples,
648        size_t numSubSamples,
649        const uint8_t key[16],
650        const uint8_t iv[16],
651        CryptoPlugin::Mode mode,
652        int64_t presentationTimeUs,
653        uint32_t flags,
654        AString *errorDetailMsg) {
655    if (errorDetailMsg != NULL) {
656        errorDetailMsg->clear();
657    }
658
659    sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
660    msg->setSize("index", index);
661    msg->setSize("offset", offset);
662    msg->setPointer("subSamples", (void *)subSamples);
663    msg->setSize("numSubSamples", numSubSamples);
664    msg->setPointer("key", (void *)key);
665    msg->setPointer("iv", (void *)iv);
666    msg->setInt32("mode", mode);
667    msg->setInt64("timeUs", presentationTimeUs);
668    msg->setInt32("flags", flags);
669    msg->setPointer("errorDetailMsg", errorDetailMsg);
670
671    sp<AMessage> response;
672    status_t err = PostAndAwaitResponse(msg, &response);
673
674    return err;
675}
676
677status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) {
678    sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, this);
679    msg->setInt64("timeoutUs", timeoutUs);
680
681    sp<AMessage> response;
682    status_t err;
683    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
684        return err;
685    }
686
687    CHECK(response->findSize("index", index));
688
689    return OK;
690}
691
692status_t MediaCodec::dequeueOutputBuffer(
693        size_t *index,
694        size_t *offset,
695        size_t *size,
696        int64_t *presentationTimeUs,
697        uint32_t *flags,
698        int64_t timeoutUs) {
699    sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, this);
700    msg->setInt64("timeoutUs", timeoutUs);
701
702    sp<AMessage> response;
703    status_t err;
704    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
705        return err;
706    }
707
708    CHECK(response->findSize("index", index));
709    CHECK(response->findSize("offset", offset));
710    CHECK(response->findSize("size", size));
711    CHECK(response->findInt64("timeUs", presentationTimeUs));
712    CHECK(response->findInt32("flags", (int32_t *)flags));
713
714    return OK;
715}
716
717status_t MediaCodec::renderOutputBufferAndRelease(size_t index) {
718    sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
719    msg->setSize("index", index);
720    msg->setInt32("render", true);
721
722    sp<AMessage> response;
723    return PostAndAwaitResponse(msg, &response);
724}
725
726status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) {
727    sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
728    msg->setSize("index", index);
729    msg->setInt32("render", true);
730    msg->setInt64("timestampNs", timestampNs);
731
732    sp<AMessage> response;
733    return PostAndAwaitResponse(msg, &response);
734}
735
736status_t MediaCodec::releaseOutputBuffer(size_t index) {
737    sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
738    msg->setSize("index", index);
739
740    sp<AMessage> response;
741    return PostAndAwaitResponse(msg, &response);
742}
743
744status_t MediaCodec::signalEndOfInputStream() {
745    sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, this);
746
747    sp<AMessage> response;
748    return PostAndAwaitResponse(msg, &response);
749}
750
751status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const {
752    sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, this);
753
754    sp<AMessage> response;
755    status_t err;
756    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
757        return err;
758    }
759
760    CHECK(response->findMessage("format", format));
761
762    return OK;
763}
764
765status_t MediaCodec::getInputFormat(sp<AMessage> *format) const {
766    sp<AMessage> msg = new AMessage(kWhatGetInputFormat, this);
767
768    sp<AMessage> response;
769    status_t err;
770    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
771        return err;
772    }
773
774    CHECK(response->findMessage("format", format));
775
776    return OK;
777}
778
779status_t MediaCodec::getName(AString *name) const {
780    sp<AMessage> msg = new AMessage(kWhatGetName, this);
781
782    sp<AMessage> response;
783    status_t err;
784    if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
785        return err;
786    }
787
788    CHECK(response->findString("name", name));
789
790    return OK;
791}
792
793status_t MediaCodec::getWidevineLegacyBuffers(Vector<sp<ABuffer> > *buffers) const {
794    sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
795    msg->setInt32("portIndex", kPortIndexInput);
796    msg->setPointer("buffers", buffers);
797    msg->setInt32("widevine", true);
798
799    sp<AMessage> response;
800    return PostAndAwaitResponse(msg, &response);
801}
802
803status_t MediaCodec::getInputBuffers(Vector<sp<ABuffer> > *buffers) const {
804    sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
805    msg->setInt32("portIndex", kPortIndexInput);
806    msg->setPointer("buffers", buffers);
807
808    sp<AMessage> response;
809    return PostAndAwaitResponse(msg, &response);
810}
811
812status_t MediaCodec::getOutputBuffers(Vector<sp<ABuffer> > *buffers) const {
813    sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
814    msg->setInt32("portIndex", kPortIndexOutput);
815    msg->setPointer("buffers", buffers);
816
817    sp<AMessage> response;
818    return PostAndAwaitResponse(msg, &response);
819}
820
821status_t MediaCodec::getOutputBuffer(size_t index, sp<ABuffer> *buffer) {
822    sp<AMessage> format;
823    return getBufferAndFormat(kPortIndexOutput, index, buffer, &format);
824}
825
826status_t MediaCodec::getOutputFormat(size_t index, sp<AMessage> *format) {
827    sp<ABuffer> buffer;
828    return getBufferAndFormat(kPortIndexOutput, index, &buffer, format);
829}
830
831status_t MediaCodec::getInputBuffer(size_t index, sp<ABuffer> *buffer) {
832    sp<AMessage> format;
833    return getBufferAndFormat(kPortIndexInput, index, buffer, &format);
834}
835
836bool MediaCodec::isExecuting() const {
837    return mState == STARTED || mState == FLUSHED;
838}
839
840status_t MediaCodec::getBufferAndFormat(
841        size_t portIndex, size_t index,
842        sp<ABuffer> *buffer, sp<AMessage> *format) {
843    // use mutex instead of a context switch
844
845    if (mReleasedByResourceManager) {
846        return DEAD_OBJECT;
847    }
848
849    buffer->clear();
850    format->clear();
851    if (!isExecuting()) {
852        return INVALID_OPERATION;
853    }
854
855    // we do not want mPortBuffers to change during this section
856    // we also don't want mOwnedByClient to change during this
857    Mutex::Autolock al(mBufferLock);
858    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
859    if (index < buffers->size()) {
860        const BufferInfo &info = buffers->itemAt(index);
861        if (info.mOwnedByClient) {
862            // by the time buffers array is initialized, crypto is set
863            if (portIndex == kPortIndexInput && mCrypto != NULL) {
864                *buffer = info.mEncryptedData;
865            } else {
866                *buffer = info.mData;
867            }
868            *format = info.mFormat;
869        }
870    }
871    return OK;
872}
873
874status_t MediaCodec::flush() {
875    sp<AMessage> msg = new AMessage(kWhatFlush, this);
876
877    sp<AMessage> response;
878    return PostAndAwaitResponse(msg, &response);
879}
880
881status_t MediaCodec::requestIDRFrame() {
882    (new AMessage(kWhatRequestIDRFrame, this))->post();
883
884    return OK;
885}
886
887void MediaCodec::requestActivityNotification(const sp<AMessage> &notify) {
888    sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, this);
889    msg->setMessage("notify", notify);
890    msg->post();
891}
892
893////////////////////////////////////////////////////////////////////////////////
894
895void MediaCodec::cancelPendingDequeueOperations() {
896    if (mFlags & kFlagDequeueInputPending) {
897        PostReplyWithError(mDequeueInputReplyID, INVALID_OPERATION);
898
899        ++mDequeueInputTimeoutGeneration;
900        mDequeueInputReplyID = 0;
901        mFlags &= ~kFlagDequeueInputPending;
902    }
903
904    if (mFlags & kFlagDequeueOutputPending) {
905        PostReplyWithError(mDequeueOutputReplyID, INVALID_OPERATION);
906
907        ++mDequeueOutputTimeoutGeneration;
908        mDequeueOutputReplyID = 0;
909        mFlags &= ~kFlagDequeueOutputPending;
910    }
911}
912
913bool MediaCodec::handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest) {
914    if (!isExecuting() || (mFlags & kFlagIsAsync)
915            || (newRequest && (mFlags & kFlagDequeueInputPending))) {
916        PostReplyWithError(replyID, INVALID_OPERATION);
917        return true;
918    } else if (mFlags & kFlagStickyError) {
919        PostReplyWithError(replyID, getStickyError());
920        return true;
921    }
922
923    ssize_t index = dequeuePortBuffer(kPortIndexInput);
924
925    if (index < 0) {
926        CHECK_EQ(index, -EAGAIN);
927        return false;
928    }
929
930    sp<AMessage> response = new AMessage;
931    response->setSize("index", index);
932    response->postReply(replyID);
933
934    return true;
935}
936
937bool MediaCodec::handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest) {
938    if (!isExecuting() || (mFlags & kFlagIsAsync)
939            || (newRequest && (mFlags & kFlagDequeueOutputPending))) {
940        PostReplyWithError(replyID, INVALID_OPERATION);
941    } else if (mFlags & kFlagStickyError) {
942        PostReplyWithError(replyID, getStickyError());
943    } else if (mFlags & kFlagOutputBuffersChanged) {
944        PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED);
945        mFlags &= ~kFlagOutputBuffersChanged;
946    } else if (mFlags & kFlagOutputFormatChanged) {
947        PostReplyWithError(replyID, INFO_FORMAT_CHANGED);
948        mFlags &= ~kFlagOutputFormatChanged;
949    } else {
950        sp<AMessage> response = new AMessage;
951        ssize_t index = dequeuePortBuffer(kPortIndexOutput);
952
953        if (index < 0) {
954            CHECK_EQ(index, -EAGAIN);
955            return false;
956        }
957
958        const sp<ABuffer> &buffer =
959            mPortBuffers[kPortIndexOutput].itemAt(index).mData;
960
961        response->setSize("index", index);
962        response->setSize("offset", buffer->offset());
963        response->setSize("size", buffer->size());
964
965        int64_t timeUs;
966        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
967
968        response->setInt64("timeUs", timeUs);
969
970        int32_t omxFlags;
971        CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags));
972
973        uint32_t flags = 0;
974        if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) {
975            flags |= BUFFER_FLAG_SYNCFRAME;
976        }
977        if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) {
978            flags |= BUFFER_FLAG_CODECCONFIG;
979        }
980        if (omxFlags & OMX_BUFFERFLAG_EOS) {
981            flags |= BUFFER_FLAG_EOS;
982        }
983
984        response->setInt32("flags", flags);
985        response->postReply(replyID);
986    }
987
988    return true;
989}
990
991void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
992    switch (msg->what()) {
993        case kWhatCodecNotify:
994        {
995            int32_t what;
996            CHECK(msg->findInt32("what", &what));
997
998            switch (what) {
999                case CodecBase::kWhatError:
1000                {
1001                    int32_t err, actionCode;
1002                    CHECK(msg->findInt32("err", &err));
1003                    CHECK(msg->findInt32("actionCode", &actionCode));
1004
1005                    ALOGE("Codec reported err %#x, actionCode %d, while in state %d",
1006                            err, actionCode, mState);
1007                    if (err == DEAD_OBJECT) {
1008                        mFlags |= kFlagSawMediaServerDie;
1009                        mFlags &= ~kFlagIsComponentAllocated;
1010                    }
1011
1012                    bool sendErrorResponse = true;
1013
1014                    switch (mState) {
1015                        case INITIALIZING:
1016                        {
1017                            setState(UNINITIALIZED);
1018                            break;
1019                        }
1020
1021                        case CONFIGURING:
1022                        {
1023                            setState(actionCode == ACTION_CODE_FATAL ?
1024                                    UNINITIALIZED : INITIALIZED);
1025                            break;
1026                        }
1027
1028                        case STARTING:
1029                        {
1030                            setState(actionCode == ACTION_CODE_FATAL ?
1031                                    UNINITIALIZED : CONFIGURED);
1032                            break;
1033                        }
1034
1035                        case STOPPING:
1036                        case RELEASING:
1037                        {
1038                            // Ignore the error, assuming we'll still get
1039                            // the shutdown complete notification.
1040
1041                            sendErrorResponse = false;
1042
1043                            if (mFlags & kFlagSawMediaServerDie) {
1044                                // MediaServer died, there definitely won't
1045                                // be a shutdown complete notification after
1046                                // all.
1047
1048                                // note that we're directly going from
1049                                // STOPPING->UNINITIALIZED, instead of the
1050                                // usual STOPPING->INITIALIZED state.
1051                                setState(UNINITIALIZED);
1052                                if (mState == RELEASING) {
1053                                    mComponentName.clear();
1054                                }
1055                                (new AMessage)->postReply(mReplyID);
1056                            }
1057                            break;
1058                        }
1059
1060                        case FLUSHING:
1061                        {
1062                            if (actionCode == ACTION_CODE_FATAL) {
1063                                setState(UNINITIALIZED);
1064                            } else {
1065                                setState(
1066                                        (mFlags & kFlagIsAsync) ? FLUSHED : STARTED);
1067                            }
1068                            break;
1069                        }
1070
1071                        case FLUSHED:
1072                        case STARTED:
1073                        {
1074                            sendErrorResponse = false;
1075
1076                            setStickyError(err);
1077                            postActivityNotificationIfPossible();
1078
1079                            cancelPendingDequeueOperations();
1080
1081                            if (mFlags & kFlagIsAsync) {
1082                                onError(err, actionCode);
1083                            }
1084                            switch (actionCode) {
1085                            case ACTION_CODE_TRANSIENT:
1086                                break;
1087                            case ACTION_CODE_RECOVERABLE:
1088                                setState(INITIALIZED);
1089                                break;
1090                            default:
1091                                setState(UNINITIALIZED);
1092                                break;
1093                            }
1094                            break;
1095                        }
1096
1097                        default:
1098                        {
1099                            sendErrorResponse = false;
1100
1101                            setStickyError(err);
1102                            postActivityNotificationIfPossible();
1103
1104                            // actionCode in an uninitialized state is always fatal.
1105                            if (mState == UNINITIALIZED) {
1106                                actionCode = ACTION_CODE_FATAL;
1107                            }
1108                            if (mFlags & kFlagIsAsync) {
1109                                onError(err, actionCode);
1110                            }
1111                            switch (actionCode) {
1112                            case ACTION_CODE_TRANSIENT:
1113                                break;
1114                            case ACTION_CODE_RECOVERABLE:
1115                                setState(INITIALIZED);
1116                                break;
1117                            default:
1118                                setState(UNINITIALIZED);
1119                                break;
1120                            }
1121                            break;
1122                        }
1123                    }
1124
1125                    if (sendErrorResponse) {
1126                        PostReplyWithError(mReplyID, err);
1127                    }
1128                    break;
1129                }
1130
1131                case CodecBase::kWhatComponentAllocated:
1132                {
1133                    CHECK_EQ(mState, INITIALIZING);
1134                    setState(INITIALIZED);
1135                    mFlags |= kFlagIsComponentAllocated;
1136
1137                    CHECK(msg->findString("componentName", &mComponentName));
1138
1139                    if (mComponentName.startsWith("OMX.google.")) {
1140                        mFlags |= kFlagUsesSoftwareRenderer;
1141                    } else {
1142                        mFlags &= ~kFlagUsesSoftwareRenderer;
1143                    }
1144
1145                    String8 resourceType;
1146                    if (mComponentName.endsWith(".secure")) {
1147                        mFlags |= kFlagIsSecure;
1148                        resourceType = String8(kResourceSecureCodec);
1149                    } else {
1150                        mFlags &= ~kFlagIsSecure;
1151                        resourceType = String8(kResourceNonSecureCodec);
1152                    }
1153
1154                    const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec;
1155                    addResource(resourceType, String8(subtype), 1);
1156
1157                    (new AMessage)->postReply(mReplyID);
1158                    break;
1159                }
1160
1161                case CodecBase::kWhatComponentConfigured:
1162                {
1163                    if (mState == UNINITIALIZED || mState == INITIALIZED) {
1164                        // In case a kWhatError message came in and replied with error,
1165                        // we log a warning and ignore.
1166                        ALOGW("configure interrupted by error, current state %d", mState);
1167                        break;
1168                    }
1169                    CHECK_EQ(mState, CONFIGURING);
1170
1171                    // reset input surface flag
1172                    mHaveInputSurface = false;
1173
1174                    CHECK(msg->findMessage("input-format", &mInputFormat));
1175                    CHECK(msg->findMessage("output-format", &mOutputFormat));
1176
1177                    int32_t usingSwRenderer;
1178                    if (mOutputFormat->findInt32("using-sw-renderer", &usingSwRenderer)
1179                            && usingSwRenderer) {
1180                        mFlags |= kFlagUsesSoftwareRenderer;
1181                    }
1182                    setState(CONFIGURED);
1183                    (new AMessage)->postReply(mReplyID);
1184                    break;
1185                }
1186
1187                case CodecBase::kWhatInputSurfaceCreated:
1188                {
1189                    // response to initiateCreateInputSurface()
1190                    status_t err = NO_ERROR;
1191                    sp<AMessage> response = new AMessage;
1192                    if (!msg->findInt32("err", &err)) {
1193                        sp<RefBase> obj;
1194                        msg->findObject("input-surface", &obj);
1195                        CHECK(obj != NULL);
1196                        response->setObject("input-surface", obj);
1197                        mHaveInputSurface = true;
1198                    } else {
1199                        response->setInt32("err", err);
1200                    }
1201                    response->postReply(mReplyID);
1202                    break;
1203                }
1204
1205                case CodecBase::kWhatInputSurfaceAccepted:
1206                {
1207                    // response to initiateSetInputSurface()
1208                    status_t err = NO_ERROR;
1209                    sp<AMessage> response = new AMessage();
1210                    if (!msg->findInt32("err", &err)) {
1211                        mHaveInputSurface = true;
1212                    } else {
1213                        response->setInt32("err", err);
1214                    }
1215                    response->postReply(mReplyID);
1216                    break;
1217                }
1218
1219                case CodecBase::kWhatSignaledInputEOS:
1220                {
1221                    // response to signalEndOfInputStream()
1222                    sp<AMessage> response = new AMessage;
1223                    status_t err;
1224                    if (msg->findInt32("err", &err)) {
1225                        response->setInt32("err", err);
1226                    }
1227                    response->postReply(mReplyID);
1228                    break;
1229                }
1230
1231
1232                case CodecBase::kWhatBuffersAllocated:
1233                {
1234                    Mutex::Autolock al(mBufferLock);
1235                    int32_t portIndex;
1236                    CHECK(msg->findInt32("portIndex", &portIndex));
1237
1238                    ALOGV("%s buffers allocated",
1239                          portIndex == kPortIndexInput ? "input" : "output");
1240
1241                    CHECK(portIndex == kPortIndexInput
1242                            || portIndex == kPortIndexOutput);
1243
1244                    mPortBuffers[portIndex].clear();
1245
1246                    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
1247
1248                    sp<RefBase> obj;
1249                    CHECK(msg->findObject("portDesc", &obj));
1250
1251                    sp<CodecBase::PortDescription> portDesc =
1252                        static_cast<CodecBase::PortDescription *>(obj.get());
1253
1254                    size_t numBuffers = portDesc->countBuffers();
1255
1256                    size_t totalSize = 0;
1257                    for (size_t i = 0; i < numBuffers; ++i) {
1258                        if (portIndex == kPortIndexInput && mCrypto != NULL) {
1259                            totalSize += portDesc->bufferAt(i)->capacity();
1260                        }
1261                    }
1262
1263                    if (totalSize) {
1264                        mDealer = new MemoryDealer(totalSize, "MediaCodec");
1265                    }
1266
1267                    for (size_t i = 0; i < numBuffers; ++i) {
1268                        BufferInfo info;
1269                        info.mBufferID = portDesc->bufferIDAt(i);
1270                        info.mOwnedByClient = false;
1271                        info.mData = portDesc->bufferAt(i);
1272
1273                        if (portIndex == kPortIndexInput && mCrypto != NULL) {
1274                            sp<IMemory> mem = mDealer->allocate(info.mData->capacity());
1275                            info.mEncryptedData =
1276                                new ABuffer(mem->pointer(), info.mData->capacity());
1277                            info.mSharedEncryptedBuffer = mem;
1278                        }
1279
1280                        buffers->push_back(info);
1281                    }
1282
1283                    if (portIndex == kPortIndexOutput) {
1284                        if (mState == STARTING) {
1285                            // We're always allocating output buffers after
1286                            // allocating input buffers, so this is a good
1287                            // indication that now all buffers are allocated.
1288                            if (mIsVideo) {
1289                                String8 subtype;
1290                                addResource(
1291                                        String8(kResourceGraphicMemory),
1292                                        subtype,
1293                                        getGraphicBufferSize());
1294                            }
1295                            setState(STARTED);
1296                            (new AMessage)->postReply(mReplyID);
1297                        } else {
1298                            mFlags |= kFlagOutputBuffersChanged;
1299                            postActivityNotificationIfPossible();
1300                        }
1301                    }
1302                    break;
1303                }
1304
1305                case CodecBase::kWhatOutputFormatChanged:
1306                {
1307                    ALOGV("codec output format changed");
1308
1309                    if (mSoftRenderer == NULL &&
1310                            mSurface != NULL &&
1311                            (mFlags & kFlagUsesSoftwareRenderer)) {
1312                        AString mime;
1313                        CHECK(msg->findString("mime", &mime));
1314
1315                        if (mime.startsWithIgnoreCase("video/")) {
1316                            mSoftRenderer = new SoftwareRenderer(mSurface);
1317                        }
1318                    }
1319
1320                    mOutputFormat = msg;
1321
1322                    if (mFlags & kFlagIsEncoder) {
1323                        // Before we announce the format change we should
1324                        // collect codec specific data and amend the output
1325                        // format as necessary.
1326                        mFlags |= kFlagGatherCodecSpecificData;
1327                    } else if (mFlags & kFlagIsAsync) {
1328                        onOutputFormatChanged();
1329                    } else {
1330                        mFlags |= kFlagOutputFormatChanged;
1331                        postActivityNotificationIfPossible();
1332                    }
1333
1334                    // Notify mCrypto of video resolution changes
1335                    if (mCrypto != NULL) {
1336                        int32_t left, top, right, bottom, width, height;
1337                        if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
1338                            mCrypto->notifyResolution(right - left + 1, bottom - top + 1);
1339                        } else if (mOutputFormat->findInt32("width", &width)
1340                                && mOutputFormat->findInt32("height", &height)) {
1341                            mCrypto->notifyResolution(width, height);
1342                        }
1343                    }
1344
1345                    break;
1346                }
1347
1348                case CodecBase::kWhatOutputFramesRendered:
1349                {
1350                    // ignore these in all states except running, and check that we have a
1351                    // notification set
1352                    if (mState == STARTED && mOnFrameRenderedNotification != NULL) {
1353                        sp<AMessage> notify = mOnFrameRenderedNotification->dup();
1354                        notify->setMessage("data", msg);
1355                        notify->post();
1356                    }
1357                    break;
1358                }
1359
1360                case CodecBase::kWhatFillThisBuffer:
1361                {
1362                    /* size_t index = */updateBuffers(kPortIndexInput, msg);
1363
1364                    if (mState == FLUSHING
1365                            || mState == STOPPING
1366                            || mState == RELEASING) {
1367                        returnBuffersToCodecOnPort(kPortIndexInput);
1368                        break;
1369                    }
1370
1371                    if (!mCSD.empty()) {
1372                        ssize_t index = dequeuePortBuffer(kPortIndexInput);
1373                        CHECK_GE(index, 0);
1374
1375                        // If codec specific data had been specified as
1376                        // part of the format in the call to configure and
1377                        // if there's more csd left, we submit it here
1378                        // clients only get access to input buffers once
1379                        // this data has been exhausted.
1380
1381                        status_t err = queueCSDInputBuffer(index);
1382
1383                        if (err != OK) {
1384                            ALOGE("queueCSDInputBuffer failed w/ error %d",
1385                                  err);
1386
1387                            setStickyError(err);
1388                            postActivityNotificationIfPossible();
1389
1390                            cancelPendingDequeueOperations();
1391                        }
1392                        break;
1393                    }
1394
1395                    if (mFlags & kFlagIsAsync) {
1396                        if (!mHaveInputSurface) {
1397                            if (mState == FLUSHED) {
1398                                mHavePendingInputBuffers = true;
1399                            } else {
1400                                onInputBufferAvailable();
1401                            }
1402                        }
1403                    } else if (mFlags & kFlagDequeueInputPending) {
1404                        CHECK(handleDequeueInputBuffer(mDequeueInputReplyID));
1405
1406                        ++mDequeueInputTimeoutGeneration;
1407                        mFlags &= ~kFlagDequeueInputPending;
1408                        mDequeueInputReplyID = 0;
1409                    } else {
1410                        postActivityNotificationIfPossible();
1411                    }
1412                    break;
1413                }
1414
1415                case CodecBase::kWhatDrainThisBuffer:
1416                {
1417                    /* size_t index = */updateBuffers(kPortIndexOutput, msg);
1418
1419                    if (mState == FLUSHING
1420                            || mState == STOPPING
1421                            || mState == RELEASING) {
1422                        returnBuffersToCodecOnPort(kPortIndexOutput);
1423                        break;
1424                    }
1425
1426                    sp<ABuffer> buffer;
1427                    CHECK(msg->findBuffer("buffer", &buffer));
1428
1429                    int32_t omxFlags;
1430                    CHECK(msg->findInt32("flags", &omxFlags));
1431
1432                    buffer->meta()->setInt32("omxFlags", omxFlags);
1433
1434                    if (mFlags & kFlagGatherCodecSpecificData) {
1435                        // This is the very first output buffer after a
1436                        // format change was signalled, it'll either contain
1437                        // the one piece of codec specific data we can expect
1438                        // or there won't be codec specific data.
1439                        if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) {
1440                            status_t err =
1441                                amendOutputFormatWithCodecSpecificData(buffer);
1442
1443                            if (err != OK) {
1444                                ALOGE("Codec spit out malformed codec "
1445                                      "specific data!");
1446                            }
1447                        }
1448
1449                        mFlags &= ~kFlagGatherCodecSpecificData;
1450                        if (mFlags & kFlagIsAsync) {
1451                            onOutputFormatChanged();
1452                        } else {
1453                            mFlags |= kFlagOutputFormatChanged;
1454                        }
1455                    }
1456
1457                    if (mFlags & kFlagIsAsync) {
1458                        onOutputBufferAvailable();
1459                    } else if (mFlags & kFlagDequeueOutputPending) {
1460                        CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID));
1461
1462                        ++mDequeueOutputTimeoutGeneration;
1463                        mFlags &= ~kFlagDequeueOutputPending;
1464                        mDequeueOutputReplyID = 0;
1465                    } else {
1466                        postActivityNotificationIfPossible();
1467                    }
1468
1469                    break;
1470                }
1471
1472                case CodecBase::kWhatEOS:
1473                {
1474                    // We already notify the client of this by using the
1475                    // corresponding flag in "onOutputBufferReady".
1476                    break;
1477                }
1478
1479                case CodecBase::kWhatShutdownCompleted:
1480                {
1481                    if (mState == STOPPING) {
1482                        setState(INITIALIZED);
1483                    } else {
1484                        CHECK_EQ(mState, RELEASING);
1485                        setState(UNINITIALIZED);
1486                        mComponentName.clear();
1487                    }
1488                    mFlags &= ~kFlagIsComponentAllocated;
1489
1490                    mResourceManagerService->removeResource(getId(mResourceManagerClient));
1491
1492                    (new AMessage)->postReply(mReplyID);
1493                    break;
1494                }
1495
1496                case CodecBase::kWhatFlushCompleted:
1497                {
1498                    if (mState != FLUSHING) {
1499                        ALOGW("received FlushCompleted message in state %d",
1500                                mState);
1501                        break;
1502                    }
1503
1504                    if (mFlags & kFlagIsAsync) {
1505                        setState(FLUSHED);
1506                    } else {
1507                        setState(STARTED);
1508                        mCodec->signalResume();
1509                    }
1510
1511                    (new AMessage)->postReply(mReplyID);
1512                    break;
1513                }
1514
1515                default:
1516                    TRESPASS();
1517            }
1518            break;
1519        }
1520
1521        case kWhatInit:
1522        {
1523            sp<AReplyToken> replyID;
1524            CHECK(msg->senderAwaitsResponse(&replyID));
1525
1526            if (mState != UNINITIALIZED) {
1527                PostReplyWithError(replyID, INVALID_OPERATION);
1528                break;
1529            }
1530
1531            mReplyID = replyID;
1532            setState(INITIALIZING);
1533
1534            AString name;
1535            CHECK(msg->findString("name", &name));
1536
1537            int32_t nameIsType;
1538            int32_t encoder = false;
1539            CHECK(msg->findInt32("nameIsType", &nameIsType));
1540            if (nameIsType) {
1541                CHECK(msg->findInt32("encoder", &encoder));
1542            }
1543
1544            sp<AMessage> format = new AMessage;
1545
1546            if (nameIsType) {
1547                format->setString("mime", name.c_str());
1548                format->setInt32("encoder", encoder);
1549            } else {
1550                format->setString("componentName", name.c_str());
1551            }
1552
1553            mCodec->initiateAllocateComponent(format);
1554            break;
1555        }
1556
1557        case kWhatSetNotification:
1558        {
1559            sp<AMessage> notify;
1560            if (msg->findMessage("on-frame-rendered", &notify)) {
1561                mOnFrameRenderedNotification = notify;
1562            }
1563            break;
1564        }
1565
1566        case kWhatSetCallback:
1567        {
1568            sp<AReplyToken> replyID;
1569            CHECK(msg->senderAwaitsResponse(&replyID));
1570
1571            if (mState == UNINITIALIZED
1572                    || mState == INITIALIZING
1573                    || isExecuting()) {
1574                // callback can't be set after codec is executing,
1575                // or before it's initialized (as the callback
1576                // will be cleared when it goes to INITIALIZED)
1577                PostReplyWithError(replyID, INVALID_OPERATION);
1578                break;
1579            }
1580
1581            sp<AMessage> callback;
1582            CHECK(msg->findMessage("callback", &callback));
1583
1584            mCallback = callback;
1585
1586            if (mCallback != NULL) {
1587                ALOGI("MediaCodec will operate in async mode");
1588                mFlags |= kFlagIsAsync;
1589            } else {
1590                mFlags &= ~kFlagIsAsync;
1591            }
1592
1593            sp<AMessage> response = new AMessage;
1594            response->postReply(replyID);
1595            break;
1596        }
1597
1598        case kWhatConfigure:
1599        {
1600            sp<AReplyToken> replyID;
1601            CHECK(msg->senderAwaitsResponse(&replyID));
1602
1603            if (mState != INITIALIZED) {
1604                PostReplyWithError(replyID, INVALID_OPERATION);
1605                break;
1606            }
1607
1608            sp<RefBase> obj;
1609            CHECK(msg->findObject("surface", &obj));
1610
1611            sp<AMessage> format;
1612            CHECK(msg->findMessage("format", &format));
1613
1614            int32_t push;
1615            if (msg->findInt32("push-blank-buffers-on-shutdown", &push) && push != 0) {
1616                mFlags |= kFlagPushBlankBuffersOnShutdown;
1617            }
1618
1619            if (obj != NULL) {
1620                format->setObject("native-window", obj);
1621                status_t err = handleSetSurface(static_cast<Surface *>(obj.get()));
1622                if (err != OK) {
1623                    PostReplyWithError(replyID, err);
1624                    break;
1625                }
1626            } else {
1627                handleSetSurface(NULL);
1628            }
1629
1630            mReplyID = replyID;
1631            setState(CONFIGURING);
1632
1633            void *crypto;
1634            if (!msg->findPointer("crypto", &crypto)) {
1635                crypto = NULL;
1636            }
1637
1638            mCrypto = static_cast<ICrypto *>(crypto);
1639
1640            uint32_t flags;
1641            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1642
1643            if (flags & CONFIGURE_FLAG_ENCODE) {
1644                format->setInt32("encoder", true);
1645                mFlags |= kFlagIsEncoder;
1646            }
1647
1648            extractCSD(format);
1649
1650            mCodec->initiateConfigureComponent(format);
1651            break;
1652        }
1653
1654        case kWhatSetSurface:
1655        {
1656            sp<AReplyToken> replyID;
1657            CHECK(msg->senderAwaitsResponse(&replyID));
1658
1659            status_t err = OK;
1660            sp<Surface> surface;
1661
1662            switch (mState) {
1663                case CONFIGURED:
1664                case STARTED:
1665                case FLUSHED:
1666                {
1667                    sp<RefBase> obj;
1668                    (void)msg->findObject("surface", &obj);
1669                    sp<Surface> surface = static_cast<Surface *>(obj.get());
1670                    if (mSurface == NULL) {
1671                        // do not support setting surface if it was not set
1672                        err = INVALID_OPERATION;
1673                    } else if (obj == NULL) {
1674                        // do not support unsetting surface
1675                        err = BAD_VALUE;
1676                    } else {
1677                        err = connectToSurface(surface);
1678                        if (err == BAD_VALUE) {
1679                            // assuming reconnecting to same surface
1680                            // TODO: check if it is the same surface
1681                            err = OK;
1682                        } else {
1683                            if (err == OK) {
1684                                if (mFlags & kFlagUsesSoftwareRenderer) {
1685                                    if (mSoftRenderer != NULL
1686                                            && (mFlags & kFlagPushBlankBuffersOnShutdown)) {
1687                                        pushBlankBuffersToNativeWindow(mSurface.get());
1688                                    }
1689                                    mSoftRenderer = new SoftwareRenderer(surface);
1690                                    // TODO: check if this was successful
1691                                } else {
1692                                    err = mCodec->setSurface(surface);
1693                                }
1694                            }
1695                            if (err == OK) {
1696                                (void)disconnectFromSurface();
1697                                mSurface = surface;
1698                            }
1699                        }
1700                    }
1701                    break;
1702                }
1703
1704                default:
1705                    err = INVALID_OPERATION;
1706                    break;
1707            }
1708
1709            PostReplyWithError(replyID, err);
1710            break;
1711        }
1712
1713        case kWhatCreateInputSurface:
1714        case kWhatSetInputSurface:
1715        {
1716            sp<AReplyToken> replyID;
1717            CHECK(msg->senderAwaitsResponse(&replyID));
1718
1719            // Must be configured, but can't have been started yet.
1720            if (mState != CONFIGURED) {
1721                PostReplyWithError(replyID, INVALID_OPERATION);
1722                break;
1723            }
1724
1725            mReplyID = replyID;
1726            if (msg->what() == kWhatCreateInputSurface) {
1727                mCodec->initiateCreateInputSurface();
1728            } else {
1729                sp<RefBase> obj;
1730                CHECK(msg->findObject("input-surface", &obj));
1731
1732                mCodec->initiateSetInputSurface(
1733                        static_cast<PersistentSurface *>(obj.get()));
1734            }
1735            break;
1736        }
1737        case kWhatStart:
1738        {
1739            sp<AReplyToken> replyID;
1740            CHECK(msg->senderAwaitsResponse(&replyID));
1741
1742            if (mState == FLUSHED) {
1743                setState(STARTED);
1744                if (mHavePendingInputBuffers) {
1745                    onInputBufferAvailable();
1746                    mHavePendingInputBuffers = false;
1747                }
1748                mCodec->signalResume();
1749                PostReplyWithError(replyID, OK);
1750                break;
1751            } else if (mState != CONFIGURED) {
1752                PostReplyWithError(replyID, INVALID_OPERATION);
1753                break;
1754            }
1755
1756            mReplyID = replyID;
1757            setState(STARTING);
1758
1759            mCodec->initiateStart();
1760            break;
1761        }
1762
1763        case kWhatStop:
1764        case kWhatRelease:
1765        {
1766            State targetState =
1767                (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED;
1768
1769            sp<AReplyToken> replyID;
1770            CHECK(msg->senderAwaitsResponse(&replyID));
1771
1772            // already stopped/released
1773            if (mState == UNINITIALIZED && mReleasedByResourceManager) {
1774                sp<AMessage> response = new AMessage;
1775                response->setInt32("err", OK);
1776                response->postReply(replyID);
1777                break;
1778            }
1779
1780            int32_t reclaimed = 0;
1781            msg->findInt32("reclaimed", &reclaimed);
1782            if (reclaimed) {
1783                mReleasedByResourceManager = true;
1784            }
1785
1786            if (!((mFlags & kFlagIsComponentAllocated) && targetState == UNINITIALIZED) // See 1
1787                    && mState != INITIALIZED
1788                    && mState != CONFIGURED && !isExecuting()) {
1789                // 1) Permit release to shut down the component if allocated.
1790                //
1791                // 2) We may be in "UNINITIALIZED" state already and
1792                // also shutdown the encoder/decoder without the
1793                // client being aware of this if media server died while
1794                // we were being stopped. The client would assume that
1795                // after stop() returned, it would be safe to call release()
1796                // and it should be in this case, no harm to allow a release()
1797                // if we're already uninitialized.
1798                sp<AMessage> response = new AMessage;
1799                // TODO: we shouldn't throw an exception for stop/release. Change this to wait until
1800                // the previous stop/release completes and then reply with OK.
1801                status_t err = mState == targetState ? OK : INVALID_OPERATION;
1802                response->setInt32("err", err);
1803                if (err == OK && targetState == UNINITIALIZED) {
1804                    mComponentName.clear();
1805                }
1806                response->postReply(replyID);
1807                break;
1808            }
1809
1810            if (mFlags & kFlagSawMediaServerDie) {
1811                // It's dead, Jim. Don't expect initiateShutdown to yield
1812                // any useful results now...
1813                setState(UNINITIALIZED);
1814                if (targetState == UNINITIALIZED) {
1815                    mComponentName.clear();
1816                }
1817                (new AMessage)->postReply(replyID);
1818                break;
1819            }
1820
1821            mReplyID = replyID;
1822            setState(msg->what() == kWhatStop ? STOPPING : RELEASING);
1823
1824            mCodec->initiateShutdown(
1825                    msg->what() == kWhatStop /* keepComponentAllocated */);
1826
1827            returnBuffersToCodec();
1828
1829            if (mSoftRenderer != NULL && (mFlags & kFlagPushBlankBuffersOnShutdown)) {
1830                pushBlankBuffersToNativeWindow(mSurface.get());
1831            }
1832            break;
1833        }
1834
1835        case kWhatDequeueInputBuffer:
1836        {
1837            sp<AReplyToken> replyID;
1838            CHECK(msg->senderAwaitsResponse(&replyID));
1839
1840            if (mFlags & kFlagIsAsync) {
1841                ALOGE("dequeueOutputBuffer can't be used in async mode");
1842                PostReplyWithError(replyID, INVALID_OPERATION);
1843                break;
1844            }
1845
1846            if (mHaveInputSurface) {
1847                ALOGE("dequeueInputBuffer can't be used with input surface");
1848                PostReplyWithError(replyID, INVALID_OPERATION);
1849                break;
1850            }
1851
1852            if (handleDequeueInputBuffer(replyID, true /* new request */)) {
1853                break;
1854            }
1855
1856            int64_t timeoutUs;
1857            CHECK(msg->findInt64("timeoutUs", &timeoutUs));
1858
1859            if (timeoutUs == 0ll) {
1860                PostReplyWithError(replyID, -EAGAIN);
1861                break;
1862            }
1863
1864            mFlags |= kFlagDequeueInputPending;
1865            mDequeueInputReplyID = replyID;
1866
1867            if (timeoutUs > 0ll) {
1868                sp<AMessage> timeoutMsg =
1869                    new AMessage(kWhatDequeueInputTimedOut, this);
1870                timeoutMsg->setInt32(
1871                        "generation", ++mDequeueInputTimeoutGeneration);
1872                timeoutMsg->post(timeoutUs);
1873            }
1874            break;
1875        }
1876
1877        case kWhatDequeueInputTimedOut:
1878        {
1879            int32_t generation;
1880            CHECK(msg->findInt32("generation", &generation));
1881
1882            if (generation != mDequeueInputTimeoutGeneration) {
1883                // Obsolete
1884                break;
1885            }
1886
1887            CHECK(mFlags & kFlagDequeueInputPending);
1888
1889            PostReplyWithError(mDequeueInputReplyID, -EAGAIN);
1890
1891            mFlags &= ~kFlagDequeueInputPending;
1892            mDequeueInputReplyID = 0;
1893            break;
1894        }
1895
1896        case kWhatQueueInputBuffer:
1897        {
1898            sp<AReplyToken> replyID;
1899            CHECK(msg->senderAwaitsResponse(&replyID));
1900
1901            if (!isExecuting()) {
1902                PostReplyWithError(replyID, INVALID_OPERATION);
1903                break;
1904            } else if (mFlags & kFlagStickyError) {
1905                PostReplyWithError(replyID, getStickyError());
1906                break;
1907            }
1908
1909            status_t err = onQueueInputBuffer(msg);
1910
1911            PostReplyWithError(replyID, err);
1912            break;
1913        }
1914
1915        case kWhatDequeueOutputBuffer:
1916        {
1917            sp<AReplyToken> replyID;
1918            CHECK(msg->senderAwaitsResponse(&replyID));
1919
1920            if (mFlags & kFlagIsAsync) {
1921                ALOGE("dequeueOutputBuffer can't be used in async mode");
1922                PostReplyWithError(replyID, INVALID_OPERATION);
1923                break;
1924            }
1925
1926            if (handleDequeueOutputBuffer(replyID, true /* new request */)) {
1927                break;
1928            }
1929
1930            int64_t timeoutUs;
1931            CHECK(msg->findInt64("timeoutUs", &timeoutUs));
1932
1933            if (timeoutUs == 0ll) {
1934                PostReplyWithError(replyID, -EAGAIN);
1935                break;
1936            }
1937
1938            mFlags |= kFlagDequeueOutputPending;
1939            mDequeueOutputReplyID = replyID;
1940
1941            if (timeoutUs > 0ll) {
1942                sp<AMessage> timeoutMsg =
1943                    new AMessage(kWhatDequeueOutputTimedOut, this);
1944                timeoutMsg->setInt32(
1945                        "generation", ++mDequeueOutputTimeoutGeneration);
1946                timeoutMsg->post(timeoutUs);
1947            }
1948            break;
1949        }
1950
1951        case kWhatDequeueOutputTimedOut:
1952        {
1953            int32_t generation;
1954            CHECK(msg->findInt32("generation", &generation));
1955
1956            if (generation != mDequeueOutputTimeoutGeneration) {
1957                // Obsolete
1958                break;
1959            }
1960
1961            CHECK(mFlags & kFlagDequeueOutputPending);
1962
1963            PostReplyWithError(mDequeueOutputReplyID, -EAGAIN);
1964
1965            mFlags &= ~kFlagDequeueOutputPending;
1966            mDequeueOutputReplyID = 0;
1967            break;
1968        }
1969
1970        case kWhatReleaseOutputBuffer:
1971        {
1972            sp<AReplyToken> replyID;
1973            CHECK(msg->senderAwaitsResponse(&replyID));
1974
1975            if (!isExecuting()) {
1976                PostReplyWithError(replyID, INVALID_OPERATION);
1977                break;
1978            } else if (mFlags & kFlagStickyError) {
1979                PostReplyWithError(replyID, getStickyError());
1980                break;
1981            }
1982
1983            status_t err = onReleaseOutputBuffer(msg);
1984
1985            PostReplyWithError(replyID, err);
1986            break;
1987        }
1988
1989        case kWhatSignalEndOfInputStream:
1990        {
1991            sp<AReplyToken> replyID;
1992            CHECK(msg->senderAwaitsResponse(&replyID));
1993
1994            if (!isExecuting()) {
1995                PostReplyWithError(replyID, INVALID_OPERATION);
1996                break;
1997            } else if (mFlags & kFlagStickyError) {
1998                PostReplyWithError(replyID, getStickyError());
1999                break;
2000            }
2001
2002            mReplyID = replyID;
2003            mCodec->signalEndOfInputStream();
2004            break;
2005        }
2006
2007        case kWhatGetBuffers:
2008        {
2009            sp<AReplyToken> replyID;
2010            CHECK(msg->senderAwaitsResponse(&replyID));
2011            // Unfortunately widevine legacy source requires knowing all of the
2012            // codec input buffers, so we have to provide them even in async mode.
2013            int32_t widevine = 0;
2014            msg->findInt32("widevine", &widevine);
2015
2016            if (!isExecuting() || ((mFlags & kFlagIsAsync) && !widevine)) {
2017                PostReplyWithError(replyID, INVALID_OPERATION);
2018                break;
2019            } else if (mFlags & kFlagStickyError) {
2020                PostReplyWithError(replyID, getStickyError());
2021                break;
2022            }
2023
2024            int32_t portIndex;
2025            CHECK(msg->findInt32("portIndex", &portIndex));
2026
2027            Vector<sp<ABuffer> > *dstBuffers;
2028            CHECK(msg->findPointer("buffers", (void **)&dstBuffers));
2029
2030            dstBuffers->clear();
2031            const Vector<BufferInfo> &srcBuffers = mPortBuffers[portIndex];
2032
2033            for (size_t i = 0; i < srcBuffers.size(); ++i) {
2034                const BufferInfo &info = srcBuffers.itemAt(i);
2035
2036                dstBuffers->push_back(
2037                        (portIndex == kPortIndexInput && mCrypto != NULL)
2038                                ? info.mEncryptedData : info.mData);
2039            }
2040
2041            (new AMessage)->postReply(replyID);
2042            break;
2043        }
2044
2045        case kWhatFlush:
2046        {
2047            sp<AReplyToken> replyID;
2048            CHECK(msg->senderAwaitsResponse(&replyID));
2049
2050            if (!isExecuting()) {
2051                PostReplyWithError(replyID, INVALID_OPERATION);
2052                break;
2053            } else if (mFlags & kFlagStickyError) {
2054                PostReplyWithError(replyID, getStickyError());
2055                break;
2056            }
2057
2058            mReplyID = replyID;
2059            // TODO: skip flushing if already FLUSHED
2060            setState(FLUSHING);
2061
2062            mCodec->signalFlush();
2063            returnBuffersToCodec();
2064            break;
2065        }
2066
2067        case kWhatGetInputFormat:
2068        case kWhatGetOutputFormat:
2069        {
2070            sp<AMessage> format =
2071                (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat);
2072
2073            sp<AReplyToken> replyID;
2074            CHECK(msg->senderAwaitsResponse(&replyID));
2075
2076            if ((mState != CONFIGURED && mState != STARTING &&
2077                 mState != STARTED && mState != FLUSHING &&
2078                 mState != FLUSHED)
2079                    || format == NULL) {
2080                PostReplyWithError(replyID, INVALID_OPERATION);
2081                break;
2082            } else if (mFlags & kFlagStickyError) {
2083                PostReplyWithError(replyID, getStickyError());
2084                break;
2085            }
2086
2087            sp<AMessage> response = new AMessage;
2088            response->setMessage("format", format);
2089            response->postReply(replyID);
2090            break;
2091        }
2092
2093        case kWhatRequestIDRFrame:
2094        {
2095            mCodec->signalRequestIDRFrame();
2096            break;
2097        }
2098
2099        case kWhatRequestActivityNotification:
2100        {
2101            CHECK(mActivityNotify == NULL);
2102            CHECK(msg->findMessage("notify", &mActivityNotify));
2103
2104            postActivityNotificationIfPossible();
2105            break;
2106        }
2107
2108        case kWhatGetName:
2109        {
2110            sp<AReplyToken> replyID;
2111            CHECK(msg->senderAwaitsResponse(&replyID));
2112
2113            if (mComponentName.empty()) {
2114                PostReplyWithError(replyID, INVALID_OPERATION);
2115                break;
2116            }
2117
2118            sp<AMessage> response = new AMessage;
2119            response->setString("name", mComponentName.c_str());
2120            response->postReply(replyID);
2121            break;
2122        }
2123
2124        case kWhatSetParameters:
2125        {
2126            sp<AReplyToken> replyID;
2127            CHECK(msg->senderAwaitsResponse(&replyID));
2128
2129            sp<AMessage> params;
2130            CHECK(msg->findMessage("params", &params));
2131
2132            status_t err = onSetParameters(params);
2133
2134            PostReplyWithError(replyID, err);
2135            break;
2136        }
2137
2138        default:
2139            TRESPASS();
2140    }
2141}
2142
2143void MediaCodec::extractCSD(const sp<AMessage> &format) {
2144    mCSD.clear();
2145
2146    size_t i = 0;
2147    for (;;) {
2148        sp<ABuffer> csd;
2149        if (!format->findBuffer(AStringPrintf("csd-%u", i).c_str(), &csd)) {
2150            break;
2151        }
2152
2153        mCSD.push_back(csd);
2154        ++i;
2155    }
2156
2157    ALOGV("Found %zu pieces of codec specific data.", mCSD.size());
2158}
2159
2160status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) {
2161    CHECK(!mCSD.empty());
2162
2163    const BufferInfo *info =
2164        &mPortBuffers[kPortIndexInput].itemAt(bufferIndex);
2165
2166    sp<ABuffer> csd = *mCSD.begin();
2167    mCSD.erase(mCSD.begin());
2168
2169    const sp<ABuffer> &codecInputData =
2170        (mCrypto != NULL) ? info->mEncryptedData : info->mData;
2171
2172    if (csd->size() > codecInputData->capacity()) {
2173        return -EINVAL;
2174    }
2175
2176    memcpy(codecInputData->data(), csd->data(), csd->size());
2177
2178    AString errorDetailMsg;
2179
2180    sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
2181    msg->setSize("index", bufferIndex);
2182    msg->setSize("offset", 0);
2183    msg->setSize("size", csd->size());
2184    msg->setInt64("timeUs", 0ll);
2185    msg->setInt32("flags", BUFFER_FLAG_CODECCONFIG);
2186    msg->setPointer("errorDetailMsg", &errorDetailMsg);
2187
2188    return onQueueInputBuffer(msg);
2189}
2190
2191void MediaCodec::setState(State newState) {
2192    if (newState == INITIALIZED || newState == UNINITIALIZED) {
2193        delete mSoftRenderer;
2194        mSoftRenderer = NULL;
2195
2196        mCrypto.clear();
2197        handleSetSurface(NULL);
2198
2199        mInputFormat.clear();
2200        mOutputFormat.clear();
2201        mFlags &= ~kFlagOutputFormatChanged;
2202        mFlags &= ~kFlagOutputBuffersChanged;
2203        mFlags &= ~kFlagStickyError;
2204        mFlags &= ~kFlagIsEncoder;
2205        mFlags &= ~kFlagGatherCodecSpecificData;
2206        mFlags &= ~kFlagIsAsync;
2207        mStickyError = OK;
2208
2209        mActivityNotify.clear();
2210        mCallback.clear();
2211    }
2212
2213    if (newState == UNINITIALIZED) {
2214        // return any straggling buffers, e.g. if we got here on an error
2215        returnBuffersToCodec();
2216
2217        // The component is gone, mediaserver's probably back up already
2218        // but should definitely be back up should we try to instantiate
2219        // another component.. and the cycle continues.
2220        mFlags &= ~kFlagSawMediaServerDie;
2221    }
2222
2223    mState = newState;
2224
2225    cancelPendingDequeueOperations();
2226
2227    updateBatteryStat();
2228}
2229
2230void MediaCodec::returnBuffersToCodec() {
2231    returnBuffersToCodecOnPort(kPortIndexInput);
2232    returnBuffersToCodecOnPort(kPortIndexOutput);
2233}
2234
2235void MediaCodec::returnBuffersToCodecOnPort(int32_t portIndex) {
2236    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
2237    Mutex::Autolock al(mBufferLock);
2238
2239    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
2240
2241    for (size_t i = 0; i < buffers->size(); ++i) {
2242        BufferInfo *info = &buffers->editItemAt(i);
2243
2244        if (info->mNotify != NULL) {
2245            sp<AMessage> msg = info->mNotify;
2246            info->mNotify = NULL;
2247            info->mOwnedByClient = false;
2248
2249            if (portIndex == kPortIndexInput) {
2250                /* no error, just returning buffers */
2251                msg->setInt32("err", OK);
2252            }
2253            msg->post();
2254        }
2255    }
2256
2257    mAvailPortBuffers[portIndex].clear();
2258}
2259
2260size_t MediaCodec::updateBuffers(
2261        int32_t portIndex, const sp<AMessage> &msg) {
2262    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
2263
2264    uint32_t bufferID;
2265    CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID));
2266
2267    Vector<BufferInfo> *buffers = &mPortBuffers[portIndex];
2268
2269    for (size_t i = 0; i < buffers->size(); ++i) {
2270        BufferInfo *info = &buffers->editItemAt(i);
2271
2272        if (info->mBufferID == bufferID) {
2273            CHECK(info->mNotify == NULL);
2274            CHECK(msg->findMessage("reply", &info->mNotify));
2275
2276            info->mFormat =
2277                (portIndex == kPortIndexInput) ? mInputFormat : mOutputFormat;
2278            mAvailPortBuffers[portIndex].push_back(i);
2279
2280            return i;
2281        }
2282    }
2283
2284    TRESPASS();
2285
2286    return 0;
2287}
2288
2289status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) {
2290    size_t index;
2291    size_t offset;
2292    size_t size;
2293    int64_t timeUs;
2294    uint32_t flags;
2295    CHECK(msg->findSize("index", &index));
2296    CHECK(msg->findSize("offset", &offset));
2297    CHECK(msg->findInt64("timeUs", &timeUs));
2298    CHECK(msg->findInt32("flags", (int32_t *)&flags));
2299
2300    const CryptoPlugin::SubSample *subSamples;
2301    size_t numSubSamples;
2302    const uint8_t *key;
2303    const uint8_t *iv;
2304    CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted;
2305
2306    // We allow the simpler queueInputBuffer API to be used even in
2307    // secure mode, by fabricating a single unencrypted subSample.
2308    CryptoPlugin::SubSample ss;
2309
2310    if (msg->findSize("size", &size)) {
2311        if (mCrypto != NULL) {
2312            ss.mNumBytesOfClearData = size;
2313            ss.mNumBytesOfEncryptedData = 0;
2314
2315            subSamples = &ss;
2316            numSubSamples = 1;
2317            key = NULL;
2318            iv = NULL;
2319        }
2320    } else {
2321        if (mCrypto == NULL) {
2322            return -EINVAL;
2323        }
2324
2325        CHECK(msg->findPointer("subSamples", (void **)&subSamples));
2326        CHECK(msg->findSize("numSubSamples", &numSubSamples));
2327        CHECK(msg->findPointer("key", (void **)&key));
2328        CHECK(msg->findPointer("iv", (void **)&iv));
2329
2330        int32_t tmp;
2331        CHECK(msg->findInt32("mode", &tmp));
2332
2333        mode = (CryptoPlugin::Mode)tmp;
2334
2335        size = 0;
2336        for (size_t i = 0; i < numSubSamples; ++i) {
2337            size += subSamples[i].mNumBytesOfClearData;
2338            size += subSamples[i].mNumBytesOfEncryptedData;
2339        }
2340    }
2341
2342    if (index >= mPortBuffers[kPortIndexInput].size()) {
2343        return -ERANGE;
2344    }
2345
2346    BufferInfo *info = &mPortBuffers[kPortIndexInput].editItemAt(index);
2347
2348    if (info->mNotify == NULL || !info->mOwnedByClient) {
2349        return -EACCES;
2350    }
2351
2352    if (offset + size > info->mData->capacity()) {
2353        return -EINVAL;
2354    }
2355
2356    sp<AMessage> reply = info->mNotify;
2357    info->mData->setRange(offset, size);
2358    info->mData->meta()->setInt64("timeUs", timeUs);
2359
2360    if (flags & BUFFER_FLAG_EOS) {
2361        info->mData->meta()->setInt32("eos", true);
2362    }
2363
2364    if (flags & BUFFER_FLAG_CODECCONFIG) {
2365        info->mData->meta()->setInt32("csd", true);
2366    }
2367
2368    if (mCrypto != NULL) {
2369        if (size > info->mEncryptedData->capacity()) {
2370            return -ERANGE;
2371        }
2372
2373        AString *errorDetailMsg;
2374        CHECK(msg->findPointer("errorDetailMsg", (void **)&errorDetailMsg));
2375
2376        ssize_t result = mCrypto->decrypt(
2377                (mFlags & kFlagIsSecure) != 0,
2378                key,
2379                iv,
2380                mode,
2381                info->mSharedEncryptedBuffer,
2382                offset,
2383                subSamples,
2384                numSubSamples,
2385                info->mData->base(),
2386                errorDetailMsg);
2387
2388        if (result < 0) {
2389            return result;
2390        }
2391
2392        info->mData->setRange(0, result);
2393    }
2394
2395    // synchronization boundary for getBufferAndFormat
2396    {
2397        Mutex::Autolock al(mBufferLock);
2398        info->mOwnedByClient = false;
2399    }
2400    reply->setBuffer("buffer", info->mData);
2401    reply->post();
2402
2403    info->mNotify = NULL;
2404
2405    return OK;
2406}
2407
2408//static
2409size_t MediaCodec::CreateFramesRenderedMessage(
2410        std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg) {
2411    size_t index = 0;
2412
2413    for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
2414            it != done.cend(); ++it) {
2415        if (it->getRenderTimeNs() < 0) {
2416            continue; // dropped frame from tracking
2417        }
2418        msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs());
2419        msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs());
2420        ++index;
2421    }
2422    return index;
2423}
2424
2425status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
2426    size_t index;
2427    CHECK(msg->findSize("index", &index));
2428
2429    int32_t render;
2430    if (!msg->findInt32("render", &render)) {
2431        render = 0;
2432    }
2433
2434    if (!isExecuting()) {
2435        return -EINVAL;
2436    }
2437
2438    if (index >= mPortBuffers[kPortIndexOutput].size()) {
2439        return -ERANGE;
2440    }
2441
2442    BufferInfo *info = &mPortBuffers[kPortIndexOutput].editItemAt(index);
2443
2444    if (info->mNotify == NULL || !info->mOwnedByClient) {
2445        return -EACCES;
2446    }
2447
2448    // synchronization boundary for getBufferAndFormat
2449    {
2450        Mutex::Autolock al(mBufferLock);
2451        info->mOwnedByClient = false;
2452    }
2453
2454    if (render && info->mData != NULL && info->mData->size() != 0) {
2455        info->mNotify->setInt32("render", true);
2456
2457        int64_t mediaTimeUs = -1;
2458        info->mData->meta()->findInt64("timeUs", &mediaTimeUs);
2459
2460        int64_t renderTimeNs = 0;
2461        if (!msg->findInt64("timestampNs", &renderTimeNs)) {
2462            // use media timestamp if client did not request a specific render timestamp
2463            ALOGV("using buffer PTS of %lld", (long long)mediaTimeUs);
2464            renderTimeNs = mediaTimeUs * 1000;
2465        }
2466        info->mNotify->setInt64("timestampNs", renderTimeNs);
2467
2468        if (mSoftRenderer != NULL) {
2469            std::list<FrameRenderTracker::Info> doneFrames = mSoftRenderer->render(
2470                    info->mData->data(), info->mData->size(),
2471                    mediaTimeUs, renderTimeNs, NULL, info->mFormat);
2472
2473            // if we are running, notify rendered frames
2474            if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) {
2475                sp<AMessage> notify = mOnFrameRenderedNotification->dup();
2476                sp<AMessage> data = new AMessage;
2477                if (CreateFramesRenderedMessage(doneFrames, data)) {
2478                    notify->setMessage("data", data);
2479                    notify->post();
2480                }
2481            }
2482        }
2483    }
2484
2485    info->mNotify->post();
2486    info->mNotify = NULL;
2487
2488    return OK;
2489}
2490
2491ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {
2492    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
2493
2494    List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];
2495
2496    if (availBuffers->empty()) {
2497        return -EAGAIN;
2498    }
2499
2500    size_t index = *availBuffers->begin();
2501    availBuffers->erase(availBuffers->begin());
2502
2503    BufferInfo *info = &mPortBuffers[portIndex].editItemAt(index);
2504    CHECK(!info->mOwnedByClient);
2505    {
2506        Mutex::Autolock al(mBufferLock);
2507        info->mOwnedByClient = true;
2508
2509        // set image-data
2510        if (info->mFormat != NULL) {
2511            sp<ABuffer> imageData;
2512            if (info->mFormat->findBuffer("image-data", &imageData)) {
2513                info->mData->meta()->setBuffer("image-data", imageData);
2514            }
2515            int32_t left, top, right, bottom;
2516            if (info->mFormat->findRect("crop", &left, &top, &right, &bottom)) {
2517                info->mData->meta()->setRect("crop-rect", left, top, right, bottom);
2518            }
2519        }
2520    }
2521
2522    return index;
2523}
2524
2525status_t MediaCodec::connectToSurface(const sp<Surface> &surface) {
2526    status_t err = OK;
2527    if (surface != NULL) {
2528        err = native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA);
2529        if (err == BAD_VALUE) {
2530            ALOGI("native window already connected. Assuming no change of surface");
2531        } else if (err != OK) {
2532            ALOGE("native_window_api_connect returned an error: %s (%d)", strerror(-err), err);
2533        }
2534    }
2535    return err;
2536}
2537
2538status_t MediaCodec::disconnectFromSurface() {
2539    status_t err = OK;
2540    if (mSurface != NULL) {
2541        err = native_window_api_disconnect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
2542        if (err != OK) {
2543            ALOGW("native_window_api_disconnect returned an error: %s (%d)", strerror(-err), err);
2544        }
2545        // assume disconnected even on error
2546        mSurface.clear();
2547    }
2548    return err;
2549}
2550
2551status_t MediaCodec::handleSetSurface(const sp<Surface> &surface) {
2552    status_t err = OK;
2553    if (mSurface != NULL) {
2554        (void)disconnectFromSurface();
2555    }
2556    if (surface != NULL) {
2557        err = connectToSurface(surface);
2558        if (err == OK) {
2559            mSurface = surface;
2560        }
2561    }
2562    return err;
2563}
2564
2565void MediaCodec::onInputBufferAvailable() {
2566    int32_t index;
2567    while ((index = dequeuePortBuffer(kPortIndexInput)) >= 0) {
2568        sp<AMessage> msg = mCallback->dup();
2569        msg->setInt32("callbackID", CB_INPUT_AVAILABLE);
2570        msg->setInt32("index", index);
2571        msg->post();
2572    }
2573}
2574
2575void MediaCodec::onOutputBufferAvailable() {
2576    int32_t index;
2577    while ((index = dequeuePortBuffer(kPortIndexOutput)) >= 0) {
2578        const sp<ABuffer> &buffer =
2579            mPortBuffers[kPortIndexOutput].itemAt(index).mData;
2580        sp<AMessage> msg = mCallback->dup();
2581        msg->setInt32("callbackID", CB_OUTPUT_AVAILABLE);
2582        msg->setInt32("index", index);
2583        msg->setSize("offset", buffer->offset());
2584        msg->setSize("size", buffer->size());
2585
2586        int64_t timeUs;
2587        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2588
2589        msg->setInt64("timeUs", timeUs);
2590
2591        int32_t omxFlags;
2592        CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags));
2593
2594        uint32_t flags = 0;
2595        if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) {
2596            flags |= BUFFER_FLAG_SYNCFRAME;
2597        }
2598        if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) {
2599            flags |= BUFFER_FLAG_CODECCONFIG;
2600        }
2601        if (omxFlags & OMX_BUFFERFLAG_EOS) {
2602            flags |= BUFFER_FLAG_EOS;
2603        }
2604
2605        msg->setInt32("flags", flags);
2606
2607        msg->post();
2608    }
2609}
2610
2611void MediaCodec::onError(status_t err, int32_t actionCode, const char *detail) {
2612    if (mCallback != NULL) {
2613        sp<AMessage> msg = mCallback->dup();
2614        msg->setInt32("callbackID", CB_ERROR);
2615        msg->setInt32("err", err);
2616        msg->setInt32("actionCode", actionCode);
2617
2618        if (detail != NULL) {
2619            msg->setString("detail", detail);
2620        }
2621
2622        msg->post();
2623    }
2624}
2625
2626void MediaCodec::onOutputFormatChanged() {
2627    if (mCallback != NULL) {
2628        sp<AMessage> msg = mCallback->dup();
2629        msg->setInt32("callbackID", CB_OUTPUT_FORMAT_CHANGED);
2630        msg->setMessage("format", mOutputFormat);
2631        msg->post();
2632    }
2633}
2634
2635
2636void MediaCodec::postActivityNotificationIfPossible() {
2637    if (mActivityNotify == NULL) {
2638        return;
2639    }
2640
2641    bool isErrorOrOutputChanged =
2642            (mFlags & (kFlagStickyError
2643                    | kFlagOutputBuffersChanged
2644                    | kFlagOutputFormatChanged));
2645
2646    if (isErrorOrOutputChanged
2647            || !mAvailPortBuffers[kPortIndexInput].empty()
2648            || !mAvailPortBuffers[kPortIndexOutput].empty()) {
2649        mActivityNotify->setInt32("input-buffers",
2650                mAvailPortBuffers[kPortIndexInput].size());
2651
2652        if (isErrorOrOutputChanged) {
2653            // we want consumer to dequeue as many times as it can
2654            mActivityNotify->setInt32("output-buffers", INT32_MAX);
2655        } else {
2656            mActivityNotify->setInt32("output-buffers",
2657                    mAvailPortBuffers[kPortIndexOutput].size());
2658        }
2659        mActivityNotify->post();
2660        mActivityNotify.clear();
2661    }
2662}
2663
2664status_t MediaCodec::setParameters(const sp<AMessage> &params) {
2665    sp<AMessage> msg = new AMessage(kWhatSetParameters, this);
2666    msg->setMessage("params", params);
2667
2668    sp<AMessage> response;
2669    return PostAndAwaitResponse(msg, &response);
2670}
2671
2672status_t MediaCodec::onSetParameters(const sp<AMessage> &params) {
2673    mCodec->signalSetParameters(params);
2674
2675    return OK;
2676}
2677
2678status_t MediaCodec::amendOutputFormatWithCodecSpecificData(
2679        const sp<ABuffer> &buffer) {
2680    AString mime;
2681    CHECK(mOutputFormat->findString("mime", &mime));
2682
2683    if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) {
2684        // Codec specific data should be SPS and PPS in a single buffer,
2685        // each prefixed by a startcode (0x00 0x00 0x00 0x01).
2686        // We separate the two and put them into the output format
2687        // under the keys "csd-0" and "csd-1".
2688
2689        unsigned csdIndex = 0;
2690
2691        const uint8_t *data = buffer->data();
2692        size_t size = buffer->size();
2693
2694        const uint8_t *nalStart;
2695        size_t nalSize;
2696        while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) {
2697            sp<ABuffer> csd = new ABuffer(nalSize + 4);
2698            memcpy(csd->data(), "\x00\x00\x00\x01", 4);
2699            memcpy(csd->data() + 4, nalStart, nalSize);
2700
2701            mOutputFormat->setBuffer(
2702                    AStringPrintf("csd-%u", csdIndex).c_str(), csd);
2703
2704            ++csdIndex;
2705        }
2706
2707        if (csdIndex != 2) {
2708            return ERROR_MALFORMED;
2709        }
2710    } else {
2711        // For everything else we just stash the codec specific data into
2712        // the output format as a single piece of csd under "csd-0".
2713        mOutputFormat->setBuffer("csd-0", buffer);
2714    }
2715
2716    return OK;
2717}
2718
2719void MediaCodec::updateBatteryStat() {
2720    if (mState == CONFIGURED && !mBatteryStatNotified) {
2721        BatteryNotifier& notifier(BatteryNotifier::getInstance());
2722
2723        if (mIsVideo) {
2724            notifier.noteStartVideo();
2725        } else {
2726            notifier.noteStartAudio();
2727        }
2728
2729        mBatteryStatNotified = true;
2730    } else if (mState == UNINITIALIZED && mBatteryStatNotified) {
2731        BatteryNotifier& notifier(BatteryNotifier::getInstance());
2732
2733        if (mIsVideo) {
2734            notifier.noteStopVideo();
2735        } else {
2736            notifier.noteStopAudio();
2737        }
2738
2739        mBatteryStatNotified = false;
2740    }
2741}
2742
2743}  // namespace android
2744