MediaCodec.cpp revision c8edf5af010ac24a99b302a18e7b84e8b4b2b783
1/* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "MediaCodec" 19#include <inttypes.h> 20 21#include "include/avc_utils.h" 22#include "include/SoftwareRenderer.h" 23 24#include <binder/IMemory.h> 25#include <binder/IPCThreadState.h> 26#include <binder/IServiceManager.h> 27#include <binder/MemoryDealer.h> 28#include <gui/BufferQueue.h> 29#include <gui/Surface.h> 30#include <media/ICrypto.h> 31#include <media/IOMX.h> 32#include <media/IResourceManagerService.h> 33#include <media/stagefright/foundation/ABuffer.h> 34#include <media/stagefright/foundation/ADebug.h> 35#include <media/stagefright/foundation/AMessage.h> 36#include <media/stagefright/foundation/AString.h> 37#include <media/stagefright/foundation/hexdump.h> 38#include <media/stagefright/ACodec.h> 39#include <media/stagefright/BufferProducerWrapper.h> 40#include <media/stagefright/MediaCodec.h> 41#include <media/stagefright/MediaCodecList.h> 42#include <media/stagefright/MediaDefs.h> 43#include <media/stagefright/MediaErrors.h> 44#include <media/stagefright/MediaFilter.h> 45#include <media/stagefright/MetaData.h> 46#include <media/stagefright/OMXClient.h> 47#include <media/stagefright/OMXCodec.h> 48#include <media/stagefright/PersistentSurface.h> 49#include <media/stagefright/SurfaceUtils.h> 50#include <mediautils/BatteryNotifier.h> 51#include <private/android_filesystem_config.h> 52#include <utils/Log.h> 53#include <utils/Singleton.h> 54 55namespace android { 56 57static inline int getCallingPid() { 58 return IPCThreadState::self()->getCallingPid(); 59} 60 61static int64_t getId(sp<IResourceManagerClient> client) { 62 return (int64_t) client.get(); 63} 64 65static bool isResourceError(status_t err) { 66 return (err == NO_MEMORY); 67} 68 69static const int kMaxRetry = 2; 70 71struct ResourceManagerClient : public BnResourceManagerClient { 72 ResourceManagerClient(MediaCodec* codec) : mMediaCodec(codec) {} 73 74 virtual bool reclaimResource() { 75 sp<MediaCodec> codec = mMediaCodec.promote(); 76 if (codec == NULL) { 77 // codec is already gone. 78 return true; 79 } 80 status_t err = codec->reclaim(); 81 if (err != OK) { 82 ALOGW("ResourceManagerClient failed to release codec with err %d", err); 83 } 84 return (err == OK); 85 } 86 87 virtual String8 getName() { 88 String8 ret; 89 sp<MediaCodec> codec = mMediaCodec.promote(); 90 if (codec == NULL) { 91 // codec is already gone. 92 return ret; 93 } 94 95 AString name; 96 if (codec->getName(&name) == OK) { 97 ret.setTo(name.c_str()); 98 } 99 return ret; 100 } 101 102protected: 103 virtual ~ResourceManagerClient() {} 104 105private: 106 wp<MediaCodec> mMediaCodec; 107 108 DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient); 109}; 110 111MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy() { 112} 113 114MediaCodec::ResourceManagerServiceProxy::~ResourceManagerServiceProxy() { 115 if (mService != NULL) { 116 IInterface::asBinder(mService)->unlinkToDeath(this); 117 } 118} 119 120void MediaCodec::ResourceManagerServiceProxy::init() { 121 sp<IServiceManager> sm = defaultServiceManager(); 122 sp<IBinder> binder = sm->getService(String16("media.resource_manager")); 123 mService = interface_cast<IResourceManagerService>(binder); 124 if (mService == NULL) { 125 ALOGE("Failed to get ResourceManagerService"); 126 return; 127 } 128 IInterface::asBinder(mService)->linkToDeath(this); 129} 130 131void MediaCodec::ResourceManagerServiceProxy::binderDied(const wp<IBinder>& /*who*/) { 132 ALOGW("ResourceManagerService died."); 133 Mutex::Autolock _l(mLock); 134 mService.clear(); 135} 136 137void MediaCodec::ResourceManagerServiceProxy::addResource( 138 int pid, 139 int64_t clientId, 140 const sp<IResourceManagerClient> client, 141 const Vector<MediaResource> &resources) { 142 Mutex::Autolock _l(mLock); 143 if (mService == NULL) { 144 return; 145 } 146 mService->addResource(pid, clientId, client, resources); 147} 148 149void MediaCodec::ResourceManagerServiceProxy::removeResource(int64_t clientId) { 150 Mutex::Autolock _l(mLock); 151 if (mService == NULL) { 152 return; 153 } 154 mService->removeResource(clientId); 155} 156 157bool MediaCodec::ResourceManagerServiceProxy::reclaimResource( 158 int callingPid, const Vector<MediaResource> &resources) { 159 Mutex::Autolock _l(mLock); 160 if (mService == NULL) { 161 return false; 162 } 163 return mService->reclaimResource(callingPid, resources); 164} 165 166// static 167sp<MediaCodec> MediaCodec::CreateByType( 168 const sp<ALooper> &looper, const char *mime, bool encoder, status_t *err) { 169 sp<MediaCodec> codec = new MediaCodec(looper); 170 171 const status_t ret = codec->init(mime, true /* nameIsType */, encoder); 172 if (err != NULL) { 173 *err = ret; 174 } 175 return ret == OK ? codec : NULL; // NULL deallocates codec. 176} 177 178// static 179sp<MediaCodec> MediaCodec::CreateByComponentName( 180 const sp<ALooper> &looper, const char *name, status_t *err) { 181 sp<MediaCodec> codec = new MediaCodec(looper); 182 183 const status_t ret = codec->init(name, false /* nameIsType */, false /* encoder */); 184 if (err != NULL) { 185 *err = ret; 186 } 187 return ret == OK ? codec : NULL; // NULL deallocates codec. 188} 189 190// static 191sp<PersistentSurface> MediaCodec::CreatePersistentInputSurface() { 192 OMXClient client; 193 CHECK_EQ(client.connect(), (status_t)OK); 194 sp<IOMX> omx = client.interface(); 195 196 const sp<IMediaCodecList> mediaCodecList = MediaCodecList::getInstance(); 197 if (mediaCodecList == NULL) { 198 ALOGE("Failed to obtain MediaCodecList!"); 199 return NULL; // if called from Java should raise IOException 200 } 201 202 AString tmp; 203 sp<AMessage> globalSettings = mediaCodecList->getGlobalSettings(); 204 if (globalSettings == NULL || !globalSettings->findString( 205 kMaxEncoderInputBuffers, &tmp)) { 206 ALOGE("Failed to get encoder input buffer count!"); 207 return NULL; 208 } 209 210 int32_t bufferCount = strtol(tmp.c_str(), NULL, 10); 211 if (bufferCount <= 0 212 || bufferCount > BufferQueue::MAX_MAX_ACQUIRED_BUFFERS) { 213 ALOGE("Encoder input buffer count is invalid!"); 214 return NULL; 215 } 216 217 sp<IGraphicBufferProducer> bufferProducer; 218 sp<IGraphicBufferConsumer> bufferConsumer; 219 220 status_t err = omx->createPersistentInputSurface( 221 &bufferProducer, &bufferConsumer); 222 223 if (err != OK) { 224 ALOGE("Failed to create persistent input surface."); 225 return NULL; 226 } 227 228 err = bufferConsumer->setMaxAcquiredBufferCount(bufferCount); 229 230 if (err != NO_ERROR) { 231 ALOGE("Unable to set BQ max acquired buffer count to %u: %d", 232 bufferCount, err); 233 return NULL; 234 } 235 236 return new PersistentSurface(bufferProducer, bufferConsumer); 237} 238 239MediaCodec::MediaCodec(const sp<ALooper> &looper) 240 : mState(UNINITIALIZED), 241 mReleasedByResourceManager(false), 242 mLooper(looper), 243 mCodec(NULL), 244 mReplyID(0), 245 mFlags(0), 246 mStickyError(OK), 247 mSoftRenderer(NULL), 248 mResourceManagerClient(new ResourceManagerClient(this)), 249 mResourceManagerService(new ResourceManagerServiceProxy()), 250 mBatteryStatNotified(false), 251 mIsVideo(false), 252 mVideoWidth(0), 253 mVideoHeight(0), 254 mDequeueInputTimeoutGeneration(0), 255 mDequeueInputReplyID(0), 256 mDequeueOutputTimeoutGeneration(0), 257 mDequeueOutputReplyID(0), 258 mHaveInputSurface(false), 259 mHavePendingInputBuffers(false) { 260} 261 262MediaCodec::~MediaCodec() { 263 CHECK_EQ(mState, UNINITIALIZED); 264 mResourceManagerService->removeResource(getId(mResourceManagerClient)); 265} 266 267// static 268status_t MediaCodec::PostAndAwaitResponse( 269 const sp<AMessage> &msg, sp<AMessage> *response) { 270 status_t err = msg->postAndAwaitResponse(response); 271 272 if (err != OK) { 273 return err; 274 } 275 276 if (!(*response)->findInt32("err", &err)) { 277 err = OK; 278 } 279 280 return err; 281} 282 283void MediaCodec::PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err) { 284 int32_t finalErr = err; 285 if (mReleasedByResourceManager) { 286 // override the err code if MediaCodec has been released by ResourceManager. 287 finalErr = DEAD_OBJECT; 288 } 289 290 sp<AMessage> response = new AMessage; 291 response->setInt32("err", finalErr); 292 response->postReply(replyID); 293} 294 295status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) { 296 mResourceManagerService->init(); 297 298 // save init parameters for reset 299 mInitName = name; 300 mInitNameIsType = nameIsType; 301 mInitIsEncoder = encoder; 302 303 // Current video decoders do not return from OMX_FillThisBuffer 304 // quickly, violating the OpenMAX specs, until that is remedied 305 // we need to invest in an extra looper to free the main event 306 // queue. 307 308 if (nameIsType || !strncasecmp(name.c_str(), "omx.", 4)) { 309 mCodec = new ACodec; 310 } else if (!nameIsType 311 && !strncasecmp(name.c_str(), "android.filter.", 15)) { 312 mCodec = new MediaFilter; 313 } else { 314 return NAME_NOT_FOUND; 315 } 316 317 bool secureCodec = false; 318 if (nameIsType && !strncasecmp(name.c_str(), "video/", 6)) { 319 mIsVideo = true; 320 } else { 321 AString tmp = name; 322 if (tmp.endsWith(".secure")) { 323 secureCodec = true; 324 tmp.erase(tmp.size() - 7, 7); 325 } 326 const sp<IMediaCodecList> mcl = MediaCodecList::getInstance(); 327 if (mcl == NULL) { 328 mCodec = NULL; // remove the codec. 329 return NO_INIT; // if called from Java should raise IOException 330 } 331 ssize_t codecIdx = mcl->findCodecByName(tmp.c_str()); 332 if (codecIdx >= 0) { 333 const sp<MediaCodecInfo> info = mcl->getCodecInfo(codecIdx); 334 Vector<AString> mimes; 335 info->getSupportedMimes(&mimes); 336 for (size_t i = 0; i < mimes.size(); i++) { 337 if (mimes[i].startsWith("video/")) { 338 mIsVideo = true; 339 break; 340 } 341 } 342 } 343 } 344 345 if (mIsVideo) { 346 // video codec needs dedicated looper 347 if (mCodecLooper == NULL) { 348 mCodecLooper = new ALooper; 349 mCodecLooper->setName("CodecLooper"); 350 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 351 } 352 353 mCodecLooper->registerHandler(mCodec); 354 } else { 355 mLooper->registerHandler(mCodec); 356 } 357 358 mLooper->registerHandler(this); 359 360 mCodec->setNotificationMessage(new AMessage(kWhatCodecNotify, this)); 361 362 sp<AMessage> msg = new AMessage(kWhatInit, this); 363 msg->setString("name", name); 364 msg->setInt32("nameIsType", nameIsType); 365 366 if (nameIsType) { 367 msg->setInt32("encoder", encoder); 368 } 369 370 status_t err; 371 Vector<MediaResource> resources; 372 const char *type = secureCodec ? kResourceSecureCodec : kResourceNonSecureCodec; 373 const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec; 374 resources.push_back(MediaResource(String8(type), String8(subtype), 1)); 375 for (int i = 0; i <= kMaxRetry; ++i) { 376 if (i > 0) { 377 // Don't try to reclaim resource for the first time. 378 if (!mResourceManagerService->reclaimResource(getCallingPid(), resources)) { 379 break; 380 } 381 } 382 383 sp<AMessage> response; 384 err = PostAndAwaitResponse(msg, &response); 385 if (!isResourceError(err)) { 386 break; 387 } 388 } 389 return err; 390} 391 392status_t MediaCodec::setCallback(const sp<AMessage> &callback) { 393 sp<AMessage> msg = new AMessage(kWhatSetCallback, this); 394 msg->setMessage("callback", callback); 395 396 sp<AMessage> response; 397 return PostAndAwaitResponse(msg, &response); 398} 399 400status_t MediaCodec::setOnFrameRenderedNotification(const sp<AMessage> ¬ify) { 401 sp<AMessage> msg = new AMessage(kWhatSetNotification, this); 402 msg->setMessage("on-frame-rendered", notify); 403 return msg->post(); 404} 405 406status_t MediaCodec::configure( 407 const sp<AMessage> &format, 408 const sp<Surface> &surface, 409 const sp<ICrypto> &crypto, 410 uint32_t flags) { 411 sp<AMessage> msg = new AMessage(kWhatConfigure, this); 412 413 if (mIsVideo) { 414 format->findInt32("width", &mVideoWidth); 415 format->findInt32("height", &mVideoHeight); 416 } 417 418 msg->setMessage("format", format); 419 msg->setInt32("flags", flags); 420 msg->setObject("surface", surface); 421 422 if (crypto != NULL) { 423 msg->setPointer("crypto", crypto.get()); 424 } 425 426 // save msg for reset 427 mConfigureMsg = msg; 428 429 status_t err; 430 Vector<MediaResource> resources; 431 const char *type = (mFlags & kFlagIsSecure) ? 432 kResourceSecureCodec : kResourceNonSecureCodec; 433 const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec; 434 resources.push_back(MediaResource(String8(type), String8(subtype), 1)); 435 // Don't know the buffer size at this point, but it's fine to use 1 because 436 // the reclaimResource call doesn't consider the requester's buffer size for now. 437 resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1)); 438 for (int i = 0; i <= kMaxRetry; ++i) { 439 if (i > 0) { 440 // Don't try to reclaim resource for the first time. 441 if (!mResourceManagerService->reclaimResource(getCallingPid(), resources)) { 442 break; 443 } 444 } 445 446 sp<AMessage> response; 447 err = PostAndAwaitResponse(msg, &response); 448 if (err != OK && err != INVALID_OPERATION) { 449 // MediaCodec now set state to UNINITIALIZED upon any fatal error. 450 // To maintain backward-compatibility, do a reset() to put codec 451 // back into INITIALIZED state. 452 // But don't reset if the err is INVALID_OPERATION, which means 453 // the configure failure is due to wrong state. 454 455 ALOGE("configure failed with err 0x%08x, resetting...", err); 456 reset(); 457 } 458 if (!isResourceError(err)) { 459 break; 460 } 461 } 462 return err; 463} 464 465status_t MediaCodec::setInputSurface( 466 const sp<PersistentSurface> &surface) { 467 sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this); 468 msg->setObject("input-surface", surface.get()); 469 470 sp<AMessage> response; 471 return PostAndAwaitResponse(msg, &response); 472} 473 474status_t MediaCodec::setSurface(const sp<Surface> &surface) { 475 sp<AMessage> msg = new AMessage(kWhatSetSurface, this); 476 msg->setObject("surface", surface); 477 478 sp<AMessage> response; 479 return PostAndAwaitResponse(msg, &response); 480} 481 482status_t MediaCodec::createInputSurface( 483 sp<IGraphicBufferProducer>* bufferProducer) { 484 sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, this); 485 486 sp<AMessage> response; 487 status_t err = PostAndAwaitResponse(msg, &response); 488 if (err == NO_ERROR) { 489 // unwrap the sp<IGraphicBufferProducer> 490 sp<RefBase> obj; 491 bool found = response->findObject("input-surface", &obj); 492 CHECK(found); 493 sp<BufferProducerWrapper> wrapper( 494 static_cast<BufferProducerWrapper*>(obj.get())); 495 *bufferProducer = wrapper->getBufferProducer(); 496 } else { 497 ALOGW("createInputSurface failed, err=%d", err); 498 } 499 return err; 500} 501 502uint64_t MediaCodec::getGraphicBufferSize() { 503 if (!mIsVideo) { 504 return 0; 505 } 506 507 uint64_t size = 0; 508 size_t portNum = sizeof(mPortBuffers) / sizeof((mPortBuffers)[0]); 509 for (size_t i = 0; i < portNum; ++i) { 510 // TODO: this is just an estimation, we should get the real buffer size from ACodec. 511 size += mPortBuffers[i].size() * mVideoWidth * mVideoHeight * 3 / 2; 512 } 513 return size; 514} 515 516void MediaCodec::addResource(const String8 &type, const String8 &subtype, uint64_t value) { 517 Vector<MediaResource> resources; 518 resources.push_back(MediaResource(type, subtype, value)); 519 mResourceManagerService->addResource( 520 getCallingPid(), getId(mResourceManagerClient), mResourceManagerClient, resources); 521} 522 523status_t MediaCodec::start() { 524 sp<AMessage> msg = new AMessage(kWhatStart, this); 525 526 status_t err; 527 Vector<MediaResource> resources; 528 const char *type = (mFlags & kFlagIsSecure) ? 529 kResourceSecureCodec : kResourceNonSecureCodec; 530 const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec; 531 resources.push_back(MediaResource(String8(type), String8(subtype), 1)); 532 // Don't know the buffer size at this point, but it's fine to use 1 because 533 // the reclaimResource call doesn't consider the requester's buffer size for now. 534 resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1)); 535 for (int i = 0; i <= kMaxRetry; ++i) { 536 if (i > 0) { 537 // Don't try to reclaim resource for the first time. 538 if (!mResourceManagerService->reclaimResource(getCallingPid(), resources)) { 539 break; 540 } 541 // Recover codec from previous error before retry start. 542 err = reset(); 543 if (err != OK) { 544 ALOGE("retrying start: failed to reset codec"); 545 break; 546 } 547 sp<AMessage> response; 548 err = PostAndAwaitResponse(mConfigureMsg, &response); 549 if (err != OK) { 550 ALOGE("retrying start: failed to configure codec"); 551 break; 552 } 553 } 554 555 sp<AMessage> response; 556 err = PostAndAwaitResponse(msg, &response); 557 if (!isResourceError(err)) { 558 break; 559 } 560 } 561 return err; 562} 563 564status_t MediaCodec::stop() { 565 sp<AMessage> msg = new AMessage(kWhatStop, this); 566 567 sp<AMessage> response; 568 return PostAndAwaitResponse(msg, &response); 569} 570 571status_t MediaCodec::reclaim() { 572 sp<AMessage> msg = new AMessage(kWhatRelease, this); 573 msg->setInt32("reclaimed", 1); 574 575 sp<AMessage> response; 576 return PostAndAwaitResponse(msg, &response); 577} 578 579status_t MediaCodec::release() { 580 sp<AMessage> msg = new AMessage(kWhatRelease, this); 581 582 sp<AMessage> response; 583 return PostAndAwaitResponse(msg, &response); 584} 585 586status_t MediaCodec::reset() { 587 /* When external-facing MediaCodec object is created, 588 it is already initialized. Thus, reset is essentially 589 release() followed by init(), plus clearing the state */ 590 591 status_t err = release(); 592 593 // unregister handlers 594 if (mCodec != NULL) { 595 if (mCodecLooper != NULL) { 596 mCodecLooper->unregisterHandler(mCodec->id()); 597 } else { 598 mLooper->unregisterHandler(mCodec->id()); 599 } 600 mCodec = NULL; 601 } 602 mLooper->unregisterHandler(id()); 603 604 mFlags = 0; // clear all flags 605 mStickyError = OK; 606 607 // reset state not reset by setState(UNINITIALIZED) 608 mReplyID = 0; 609 mDequeueInputReplyID = 0; 610 mDequeueOutputReplyID = 0; 611 mDequeueInputTimeoutGeneration = 0; 612 mDequeueOutputTimeoutGeneration = 0; 613 mHaveInputSurface = false; 614 615 if (err == OK) { 616 err = init(mInitName, mInitNameIsType, mInitIsEncoder); 617 } 618 return err; 619} 620 621status_t MediaCodec::queueInputBuffer( 622 size_t index, 623 size_t offset, 624 size_t size, 625 int64_t presentationTimeUs, 626 uint32_t flags, 627 AString *errorDetailMsg) { 628 if (errorDetailMsg != NULL) { 629 errorDetailMsg->clear(); 630 } 631 632 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); 633 msg->setSize("index", index); 634 msg->setSize("offset", offset); 635 msg->setSize("size", size); 636 msg->setInt64("timeUs", presentationTimeUs); 637 msg->setInt32("flags", flags); 638 msg->setPointer("errorDetailMsg", errorDetailMsg); 639 640 sp<AMessage> response; 641 return PostAndAwaitResponse(msg, &response); 642} 643 644status_t MediaCodec::queueSecureInputBuffer( 645 size_t index, 646 size_t offset, 647 const CryptoPlugin::SubSample *subSamples, 648 size_t numSubSamples, 649 const uint8_t key[16], 650 const uint8_t iv[16], 651 CryptoPlugin::Mode mode, 652 int64_t presentationTimeUs, 653 uint32_t flags, 654 AString *errorDetailMsg) { 655 if (errorDetailMsg != NULL) { 656 errorDetailMsg->clear(); 657 } 658 659 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); 660 msg->setSize("index", index); 661 msg->setSize("offset", offset); 662 msg->setPointer("subSamples", (void *)subSamples); 663 msg->setSize("numSubSamples", numSubSamples); 664 msg->setPointer("key", (void *)key); 665 msg->setPointer("iv", (void *)iv); 666 msg->setInt32("mode", mode); 667 msg->setInt64("timeUs", presentationTimeUs); 668 msg->setInt32("flags", flags); 669 msg->setPointer("errorDetailMsg", errorDetailMsg); 670 671 sp<AMessage> response; 672 status_t err = PostAndAwaitResponse(msg, &response); 673 674 return err; 675} 676 677status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) { 678 sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, this); 679 msg->setInt64("timeoutUs", timeoutUs); 680 681 sp<AMessage> response; 682 status_t err; 683 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 684 return err; 685 } 686 687 CHECK(response->findSize("index", index)); 688 689 return OK; 690} 691 692status_t MediaCodec::dequeueOutputBuffer( 693 size_t *index, 694 size_t *offset, 695 size_t *size, 696 int64_t *presentationTimeUs, 697 uint32_t *flags, 698 int64_t timeoutUs) { 699 sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, this); 700 msg->setInt64("timeoutUs", timeoutUs); 701 702 sp<AMessage> response; 703 status_t err; 704 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 705 return err; 706 } 707 708 CHECK(response->findSize("index", index)); 709 CHECK(response->findSize("offset", offset)); 710 CHECK(response->findSize("size", size)); 711 CHECK(response->findInt64("timeUs", presentationTimeUs)); 712 CHECK(response->findInt32("flags", (int32_t *)flags)); 713 714 return OK; 715} 716 717status_t MediaCodec::renderOutputBufferAndRelease(size_t index) { 718 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); 719 msg->setSize("index", index); 720 msg->setInt32("render", true); 721 722 sp<AMessage> response; 723 return PostAndAwaitResponse(msg, &response); 724} 725 726status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) { 727 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); 728 msg->setSize("index", index); 729 msg->setInt32("render", true); 730 msg->setInt64("timestampNs", timestampNs); 731 732 sp<AMessage> response; 733 return PostAndAwaitResponse(msg, &response); 734} 735 736status_t MediaCodec::releaseOutputBuffer(size_t index) { 737 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); 738 msg->setSize("index", index); 739 740 sp<AMessage> response; 741 return PostAndAwaitResponse(msg, &response); 742} 743 744status_t MediaCodec::signalEndOfInputStream() { 745 sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, this); 746 747 sp<AMessage> response; 748 return PostAndAwaitResponse(msg, &response); 749} 750 751status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const { 752 sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, this); 753 754 sp<AMessage> response; 755 status_t err; 756 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 757 return err; 758 } 759 760 CHECK(response->findMessage("format", format)); 761 762 return OK; 763} 764 765status_t MediaCodec::getInputFormat(sp<AMessage> *format) const { 766 sp<AMessage> msg = new AMessage(kWhatGetInputFormat, this); 767 768 sp<AMessage> response; 769 status_t err; 770 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 771 return err; 772 } 773 774 CHECK(response->findMessage("format", format)); 775 776 return OK; 777} 778 779status_t MediaCodec::getName(AString *name) const { 780 sp<AMessage> msg = new AMessage(kWhatGetName, this); 781 782 sp<AMessage> response; 783 status_t err; 784 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 785 return err; 786 } 787 788 CHECK(response->findString("name", name)); 789 790 return OK; 791} 792 793status_t MediaCodec::getWidevineLegacyBuffers(Vector<sp<ABuffer> > *buffers) const { 794 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this); 795 msg->setInt32("portIndex", kPortIndexInput); 796 msg->setPointer("buffers", buffers); 797 msg->setInt32("widevine", true); 798 799 sp<AMessage> response; 800 return PostAndAwaitResponse(msg, &response); 801} 802 803status_t MediaCodec::getInputBuffers(Vector<sp<ABuffer> > *buffers) const { 804 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this); 805 msg->setInt32("portIndex", kPortIndexInput); 806 msg->setPointer("buffers", buffers); 807 808 sp<AMessage> response; 809 return PostAndAwaitResponse(msg, &response); 810} 811 812status_t MediaCodec::getOutputBuffers(Vector<sp<ABuffer> > *buffers) const { 813 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this); 814 msg->setInt32("portIndex", kPortIndexOutput); 815 msg->setPointer("buffers", buffers); 816 817 sp<AMessage> response; 818 return PostAndAwaitResponse(msg, &response); 819} 820 821status_t MediaCodec::getOutputBuffer(size_t index, sp<ABuffer> *buffer) { 822 sp<AMessage> format; 823 return getBufferAndFormat(kPortIndexOutput, index, buffer, &format); 824} 825 826status_t MediaCodec::getOutputFormat(size_t index, sp<AMessage> *format) { 827 sp<ABuffer> buffer; 828 return getBufferAndFormat(kPortIndexOutput, index, &buffer, format); 829} 830 831status_t MediaCodec::getInputBuffer(size_t index, sp<ABuffer> *buffer) { 832 sp<AMessage> format; 833 return getBufferAndFormat(kPortIndexInput, index, buffer, &format); 834} 835 836bool MediaCodec::isExecuting() const { 837 return mState == STARTED || mState == FLUSHED; 838} 839 840status_t MediaCodec::getBufferAndFormat( 841 size_t portIndex, size_t index, 842 sp<ABuffer> *buffer, sp<AMessage> *format) { 843 // use mutex instead of a context switch 844 845 if (mReleasedByResourceManager) { 846 return DEAD_OBJECT; 847 } 848 849 buffer->clear(); 850 format->clear(); 851 if (!isExecuting()) { 852 return INVALID_OPERATION; 853 } 854 855 // we do not want mPortBuffers to change during this section 856 // we also don't want mOwnedByClient to change during this 857 Mutex::Autolock al(mBufferLock); 858 Vector<BufferInfo> *buffers = &mPortBuffers[portIndex]; 859 if (index < buffers->size()) { 860 const BufferInfo &info = buffers->itemAt(index); 861 if (info.mOwnedByClient) { 862 // by the time buffers array is initialized, crypto is set 863 if (portIndex == kPortIndexInput && mCrypto != NULL) { 864 *buffer = info.mEncryptedData; 865 } else { 866 *buffer = info.mData; 867 } 868 *format = info.mFormat; 869 } 870 } 871 return OK; 872} 873 874status_t MediaCodec::flush() { 875 sp<AMessage> msg = new AMessage(kWhatFlush, this); 876 877 sp<AMessage> response; 878 return PostAndAwaitResponse(msg, &response); 879} 880 881status_t MediaCodec::requestIDRFrame() { 882 (new AMessage(kWhatRequestIDRFrame, this))->post(); 883 884 return OK; 885} 886 887void MediaCodec::requestActivityNotification(const sp<AMessage> ¬ify) { 888 sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, this); 889 msg->setMessage("notify", notify); 890 msg->post(); 891} 892 893//////////////////////////////////////////////////////////////////////////////// 894 895void MediaCodec::cancelPendingDequeueOperations() { 896 if (mFlags & kFlagDequeueInputPending) { 897 PostReplyWithError(mDequeueInputReplyID, INVALID_OPERATION); 898 899 ++mDequeueInputTimeoutGeneration; 900 mDequeueInputReplyID = 0; 901 mFlags &= ~kFlagDequeueInputPending; 902 } 903 904 if (mFlags & kFlagDequeueOutputPending) { 905 PostReplyWithError(mDequeueOutputReplyID, INVALID_OPERATION); 906 907 ++mDequeueOutputTimeoutGeneration; 908 mDequeueOutputReplyID = 0; 909 mFlags &= ~kFlagDequeueOutputPending; 910 } 911} 912 913bool MediaCodec::handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest) { 914 if (!isExecuting() || (mFlags & kFlagIsAsync) 915 || (newRequest && (mFlags & kFlagDequeueInputPending))) { 916 PostReplyWithError(replyID, INVALID_OPERATION); 917 return true; 918 } else if (mFlags & kFlagStickyError) { 919 PostReplyWithError(replyID, getStickyError()); 920 return true; 921 } 922 923 ssize_t index = dequeuePortBuffer(kPortIndexInput); 924 925 if (index < 0) { 926 CHECK_EQ(index, -EAGAIN); 927 return false; 928 } 929 930 sp<AMessage> response = new AMessage; 931 response->setSize("index", index); 932 response->postReply(replyID); 933 934 return true; 935} 936 937bool MediaCodec::handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest) { 938 if (!isExecuting() || (mFlags & kFlagIsAsync) 939 || (newRequest && (mFlags & kFlagDequeueOutputPending))) { 940 PostReplyWithError(replyID, INVALID_OPERATION); 941 } else if (mFlags & kFlagStickyError) { 942 PostReplyWithError(replyID, getStickyError()); 943 } else if (mFlags & kFlagOutputBuffersChanged) { 944 PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED); 945 mFlags &= ~kFlagOutputBuffersChanged; 946 } else if (mFlags & kFlagOutputFormatChanged) { 947 PostReplyWithError(replyID, INFO_FORMAT_CHANGED); 948 mFlags &= ~kFlagOutputFormatChanged; 949 } else { 950 sp<AMessage> response = new AMessage; 951 ssize_t index = dequeuePortBuffer(kPortIndexOutput); 952 953 if (index < 0) { 954 CHECK_EQ(index, -EAGAIN); 955 return false; 956 } 957 958 const sp<ABuffer> &buffer = 959 mPortBuffers[kPortIndexOutput].itemAt(index).mData; 960 961 response->setSize("index", index); 962 response->setSize("offset", buffer->offset()); 963 response->setSize("size", buffer->size()); 964 965 int64_t timeUs; 966 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 967 968 response->setInt64("timeUs", timeUs); 969 970 int32_t omxFlags; 971 CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags)); 972 973 uint32_t flags = 0; 974 if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) { 975 flags |= BUFFER_FLAG_SYNCFRAME; 976 } 977 if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) { 978 flags |= BUFFER_FLAG_CODECCONFIG; 979 } 980 if (omxFlags & OMX_BUFFERFLAG_EOS) { 981 flags |= BUFFER_FLAG_EOS; 982 } 983 984 response->setInt32("flags", flags); 985 response->postReply(replyID); 986 } 987 988 return true; 989} 990 991void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { 992 switch (msg->what()) { 993 case kWhatCodecNotify: 994 { 995 int32_t what; 996 CHECK(msg->findInt32("what", &what)); 997 998 switch (what) { 999 case CodecBase::kWhatError: 1000 { 1001 int32_t err, actionCode; 1002 CHECK(msg->findInt32("err", &err)); 1003 CHECK(msg->findInt32("actionCode", &actionCode)); 1004 1005 ALOGE("Codec reported err %#x, actionCode %d, while in state %d", 1006 err, actionCode, mState); 1007 if (err == DEAD_OBJECT) { 1008 mFlags |= kFlagSawMediaServerDie; 1009 mFlags &= ~kFlagIsComponentAllocated; 1010 } 1011 1012 bool sendErrorResponse = true; 1013 1014 switch (mState) { 1015 case INITIALIZING: 1016 { 1017 setState(UNINITIALIZED); 1018 break; 1019 } 1020 1021 case CONFIGURING: 1022 { 1023 setState(actionCode == ACTION_CODE_FATAL ? 1024 UNINITIALIZED : INITIALIZED); 1025 break; 1026 } 1027 1028 case STARTING: 1029 { 1030 setState(actionCode == ACTION_CODE_FATAL ? 1031 UNINITIALIZED : CONFIGURED); 1032 break; 1033 } 1034 1035 case STOPPING: 1036 case RELEASING: 1037 { 1038 // Ignore the error, assuming we'll still get 1039 // the shutdown complete notification. 1040 1041 sendErrorResponse = false; 1042 1043 if (mFlags & kFlagSawMediaServerDie) { 1044 // MediaServer died, there definitely won't 1045 // be a shutdown complete notification after 1046 // all. 1047 1048 // note that we're directly going from 1049 // STOPPING->UNINITIALIZED, instead of the 1050 // usual STOPPING->INITIALIZED state. 1051 setState(UNINITIALIZED); 1052 if (mState == RELEASING) { 1053 mComponentName.clear(); 1054 } 1055 (new AMessage)->postReply(mReplyID); 1056 } 1057 break; 1058 } 1059 1060 case FLUSHING: 1061 { 1062 if (actionCode == ACTION_CODE_FATAL) { 1063 setState(UNINITIALIZED); 1064 } else { 1065 setState( 1066 (mFlags & kFlagIsAsync) ? FLUSHED : STARTED); 1067 } 1068 break; 1069 } 1070 1071 case FLUSHED: 1072 case STARTED: 1073 { 1074 sendErrorResponse = false; 1075 1076 setStickyError(err); 1077 postActivityNotificationIfPossible(); 1078 1079 cancelPendingDequeueOperations(); 1080 1081 if (mFlags & kFlagIsAsync) { 1082 onError(err, actionCode); 1083 } 1084 switch (actionCode) { 1085 case ACTION_CODE_TRANSIENT: 1086 break; 1087 case ACTION_CODE_RECOVERABLE: 1088 setState(INITIALIZED); 1089 break; 1090 default: 1091 setState(UNINITIALIZED); 1092 break; 1093 } 1094 break; 1095 } 1096 1097 default: 1098 { 1099 sendErrorResponse = false; 1100 1101 setStickyError(err); 1102 postActivityNotificationIfPossible(); 1103 1104 // actionCode in an uninitialized state is always fatal. 1105 if (mState == UNINITIALIZED) { 1106 actionCode = ACTION_CODE_FATAL; 1107 } 1108 if (mFlags & kFlagIsAsync) { 1109 onError(err, actionCode); 1110 } 1111 switch (actionCode) { 1112 case ACTION_CODE_TRANSIENT: 1113 break; 1114 case ACTION_CODE_RECOVERABLE: 1115 setState(INITIALIZED); 1116 break; 1117 default: 1118 setState(UNINITIALIZED); 1119 break; 1120 } 1121 break; 1122 } 1123 } 1124 1125 if (sendErrorResponse) { 1126 PostReplyWithError(mReplyID, err); 1127 } 1128 break; 1129 } 1130 1131 case CodecBase::kWhatComponentAllocated: 1132 { 1133 CHECK_EQ(mState, INITIALIZING); 1134 setState(INITIALIZED); 1135 mFlags |= kFlagIsComponentAllocated; 1136 1137 CHECK(msg->findString("componentName", &mComponentName)); 1138 1139 if (mComponentName.startsWith("OMX.google.")) { 1140 mFlags |= kFlagUsesSoftwareRenderer; 1141 } else { 1142 mFlags &= ~kFlagUsesSoftwareRenderer; 1143 } 1144 1145 String8 resourceType; 1146 if (mComponentName.endsWith(".secure")) { 1147 mFlags |= kFlagIsSecure; 1148 resourceType = String8(kResourceSecureCodec); 1149 } else { 1150 mFlags &= ~kFlagIsSecure; 1151 resourceType = String8(kResourceNonSecureCodec); 1152 } 1153 1154 const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec; 1155 addResource(resourceType, String8(subtype), 1); 1156 1157 (new AMessage)->postReply(mReplyID); 1158 break; 1159 } 1160 1161 case CodecBase::kWhatComponentConfigured: 1162 { 1163 CHECK_EQ(mState, CONFIGURING); 1164 1165 // reset input surface flag 1166 mHaveInputSurface = false; 1167 1168 CHECK(msg->findMessage("input-format", &mInputFormat)); 1169 CHECK(msg->findMessage("output-format", &mOutputFormat)); 1170 1171 int32_t usingSwRenderer; 1172 if (mOutputFormat->findInt32("using-sw-renderer", &usingSwRenderer) 1173 && usingSwRenderer) { 1174 mFlags |= kFlagUsesSoftwareRenderer; 1175 } 1176 setState(CONFIGURED); 1177 (new AMessage)->postReply(mReplyID); 1178 break; 1179 } 1180 1181 case CodecBase::kWhatInputSurfaceCreated: 1182 { 1183 // response to initiateCreateInputSurface() 1184 status_t err = NO_ERROR; 1185 sp<AMessage> response = new AMessage; 1186 if (!msg->findInt32("err", &err)) { 1187 sp<RefBase> obj; 1188 msg->findObject("input-surface", &obj); 1189 CHECK(obj != NULL); 1190 response->setObject("input-surface", obj); 1191 mHaveInputSurface = true; 1192 } else { 1193 response->setInt32("err", err); 1194 } 1195 response->postReply(mReplyID); 1196 break; 1197 } 1198 1199 case CodecBase::kWhatInputSurfaceAccepted: 1200 { 1201 // response to initiateSetInputSurface() 1202 status_t err = NO_ERROR; 1203 sp<AMessage> response = new AMessage(); 1204 if (!msg->findInt32("err", &err)) { 1205 mHaveInputSurface = true; 1206 } else { 1207 response->setInt32("err", err); 1208 } 1209 response->postReply(mReplyID); 1210 break; 1211 } 1212 1213 case CodecBase::kWhatSignaledInputEOS: 1214 { 1215 // response to signalEndOfInputStream() 1216 sp<AMessage> response = new AMessage; 1217 status_t err; 1218 if (msg->findInt32("err", &err)) { 1219 response->setInt32("err", err); 1220 } 1221 response->postReply(mReplyID); 1222 break; 1223 } 1224 1225 1226 case CodecBase::kWhatBuffersAllocated: 1227 { 1228 Mutex::Autolock al(mBufferLock); 1229 int32_t portIndex; 1230 CHECK(msg->findInt32("portIndex", &portIndex)); 1231 1232 ALOGV("%s buffers allocated", 1233 portIndex == kPortIndexInput ? "input" : "output"); 1234 1235 CHECK(portIndex == kPortIndexInput 1236 || portIndex == kPortIndexOutput); 1237 1238 mPortBuffers[portIndex].clear(); 1239 1240 Vector<BufferInfo> *buffers = &mPortBuffers[portIndex]; 1241 1242 sp<RefBase> obj; 1243 CHECK(msg->findObject("portDesc", &obj)); 1244 1245 sp<CodecBase::PortDescription> portDesc = 1246 static_cast<CodecBase::PortDescription *>(obj.get()); 1247 1248 size_t numBuffers = portDesc->countBuffers(); 1249 1250 size_t totalSize = 0; 1251 for (size_t i = 0; i < numBuffers; ++i) { 1252 if (portIndex == kPortIndexInput && mCrypto != NULL) { 1253 totalSize += portDesc->bufferAt(i)->capacity(); 1254 } 1255 } 1256 1257 if (totalSize) { 1258 mDealer = new MemoryDealer(totalSize, "MediaCodec"); 1259 } 1260 1261 for (size_t i = 0; i < numBuffers; ++i) { 1262 BufferInfo info; 1263 info.mBufferID = portDesc->bufferIDAt(i); 1264 info.mOwnedByClient = false; 1265 info.mData = portDesc->bufferAt(i); 1266 1267 if (portIndex == kPortIndexInput && mCrypto != NULL) { 1268 sp<IMemory> mem = mDealer->allocate(info.mData->capacity()); 1269 info.mEncryptedData = 1270 new ABuffer(mem->pointer(), info.mData->capacity()); 1271 info.mSharedEncryptedBuffer = mem; 1272 } 1273 1274 buffers->push_back(info); 1275 } 1276 1277 if (portIndex == kPortIndexOutput) { 1278 if (mState == STARTING) { 1279 // We're always allocating output buffers after 1280 // allocating input buffers, so this is a good 1281 // indication that now all buffers are allocated. 1282 if (mIsVideo) { 1283 String8 subtype; 1284 addResource( 1285 String8(kResourceGraphicMemory), 1286 subtype, 1287 getGraphicBufferSize()); 1288 } 1289 setState(STARTED); 1290 (new AMessage)->postReply(mReplyID); 1291 } else { 1292 mFlags |= kFlagOutputBuffersChanged; 1293 postActivityNotificationIfPossible(); 1294 } 1295 } 1296 break; 1297 } 1298 1299 case CodecBase::kWhatOutputFormatChanged: 1300 { 1301 ALOGV("codec output format changed"); 1302 1303 if (mSoftRenderer == NULL && 1304 mSurface != NULL && 1305 (mFlags & kFlagUsesSoftwareRenderer)) { 1306 AString mime; 1307 CHECK(msg->findString("mime", &mime)); 1308 1309 if (mime.startsWithIgnoreCase("video/")) { 1310 mSoftRenderer = new SoftwareRenderer(mSurface); 1311 } 1312 } 1313 1314 mOutputFormat = msg; 1315 1316 if (mFlags & kFlagIsEncoder) { 1317 // Before we announce the format change we should 1318 // collect codec specific data and amend the output 1319 // format as necessary. 1320 mFlags |= kFlagGatherCodecSpecificData; 1321 } else if (mFlags & kFlagIsAsync) { 1322 onOutputFormatChanged(); 1323 } else { 1324 mFlags |= kFlagOutputFormatChanged; 1325 postActivityNotificationIfPossible(); 1326 } 1327 1328 // Notify mCrypto of video resolution changes 1329 if (mCrypto != NULL) { 1330 int32_t left, top, right, bottom, width, height; 1331 if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) { 1332 mCrypto->notifyResolution(right - left + 1, bottom - top + 1); 1333 } else if (mOutputFormat->findInt32("width", &width) 1334 && mOutputFormat->findInt32("height", &height)) { 1335 mCrypto->notifyResolution(width, height); 1336 } 1337 } 1338 1339 break; 1340 } 1341 1342 case CodecBase::kWhatOutputFramesRendered: 1343 { 1344 // ignore these in all states except running, and check that we have a 1345 // notification set 1346 if (mState == STARTED && mOnFrameRenderedNotification != NULL) { 1347 sp<AMessage> notify = mOnFrameRenderedNotification->dup(); 1348 notify->setMessage("data", msg); 1349 notify->post(); 1350 } 1351 break; 1352 } 1353 1354 case CodecBase::kWhatFillThisBuffer: 1355 { 1356 /* size_t index = */updateBuffers(kPortIndexInput, msg); 1357 1358 if (mState == FLUSHING 1359 || mState == STOPPING 1360 || mState == RELEASING) { 1361 returnBuffersToCodecOnPort(kPortIndexInput); 1362 break; 1363 } 1364 1365 if (!mCSD.empty()) { 1366 ssize_t index = dequeuePortBuffer(kPortIndexInput); 1367 CHECK_GE(index, 0); 1368 1369 // If codec specific data had been specified as 1370 // part of the format in the call to configure and 1371 // if there's more csd left, we submit it here 1372 // clients only get access to input buffers once 1373 // this data has been exhausted. 1374 1375 status_t err = queueCSDInputBuffer(index); 1376 1377 if (err != OK) { 1378 ALOGE("queueCSDInputBuffer failed w/ error %d", 1379 err); 1380 1381 setStickyError(err); 1382 postActivityNotificationIfPossible(); 1383 1384 cancelPendingDequeueOperations(); 1385 } 1386 break; 1387 } 1388 1389 if (mFlags & kFlagIsAsync) { 1390 if (!mHaveInputSurface) { 1391 if (mState == FLUSHED) { 1392 mHavePendingInputBuffers = true; 1393 } else { 1394 onInputBufferAvailable(); 1395 } 1396 } 1397 } else if (mFlags & kFlagDequeueInputPending) { 1398 CHECK(handleDequeueInputBuffer(mDequeueInputReplyID)); 1399 1400 ++mDequeueInputTimeoutGeneration; 1401 mFlags &= ~kFlagDequeueInputPending; 1402 mDequeueInputReplyID = 0; 1403 } else { 1404 postActivityNotificationIfPossible(); 1405 } 1406 break; 1407 } 1408 1409 case CodecBase::kWhatDrainThisBuffer: 1410 { 1411 /* size_t index = */updateBuffers(kPortIndexOutput, msg); 1412 1413 if (mState == FLUSHING 1414 || mState == STOPPING 1415 || mState == RELEASING) { 1416 returnBuffersToCodecOnPort(kPortIndexOutput); 1417 break; 1418 } 1419 1420 sp<ABuffer> buffer; 1421 CHECK(msg->findBuffer("buffer", &buffer)); 1422 1423 int32_t omxFlags; 1424 CHECK(msg->findInt32("flags", &omxFlags)); 1425 1426 buffer->meta()->setInt32("omxFlags", omxFlags); 1427 1428 if (mFlags & kFlagGatherCodecSpecificData) { 1429 // This is the very first output buffer after a 1430 // format change was signalled, it'll either contain 1431 // the one piece of codec specific data we can expect 1432 // or there won't be codec specific data. 1433 if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) { 1434 status_t err = 1435 amendOutputFormatWithCodecSpecificData(buffer); 1436 1437 if (err != OK) { 1438 ALOGE("Codec spit out malformed codec " 1439 "specific data!"); 1440 } 1441 } 1442 1443 mFlags &= ~kFlagGatherCodecSpecificData; 1444 if (mFlags & kFlagIsAsync) { 1445 onOutputFormatChanged(); 1446 } else { 1447 mFlags |= kFlagOutputFormatChanged; 1448 } 1449 } 1450 1451 if (mFlags & kFlagIsAsync) { 1452 onOutputBufferAvailable(); 1453 } else if (mFlags & kFlagDequeueOutputPending) { 1454 CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID)); 1455 1456 ++mDequeueOutputTimeoutGeneration; 1457 mFlags &= ~kFlagDequeueOutputPending; 1458 mDequeueOutputReplyID = 0; 1459 } else { 1460 postActivityNotificationIfPossible(); 1461 } 1462 1463 break; 1464 } 1465 1466 case CodecBase::kWhatEOS: 1467 { 1468 // We already notify the client of this by using the 1469 // corresponding flag in "onOutputBufferReady". 1470 break; 1471 } 1472 1473 case CodecBase::kWhatShutdownCompleted: 1474 { 1475 if (mState == STOPPING) { 1476 setState(INITIALIZED); 1477 } else { 1478 CHECK_EQ(mState, RELEASING); 1479 setState(UNINITIALIZED); 1480 mComponentName.clear(); 1481 } 1482 mFlags &= ~kFlagIsComponentAllocated; 1483 1484 mResourceManagerService->removeResource(getId(mResourceManagerClient)); 1485 1486 (new AMessage)->postReply(mReplyID); 1487 break; 1488 } 1489 1490 case CodecBase::kWhatFlushCompleted: 1491 { 1492 if (mState != FLUSHING) { 1493 ALOGW("received FlushCompleted message in state %d", 1494 mState); 1495 break; 1496 } 1497 1498 if (mFlags & kFlagIsAsync) { 1499 setState(FLUSHED); 1500 } else { 1501 setState(STARTED); 1502 mCodec->signalResume(); 1503 } 1504 1505 (new AMessage)->postReply(mReplyID); 1506 break; 1507 } 1508 1509 default: 1510 TRESPASS(); 1511 } 1512 break; 1513 } 1514 1515 case kWhatInit: 1516 { 1517 sp<AReplyToken> replyID; 1518 CHECK(msg->senderAwaitsResponse(&replyID)); 1519 1520 if (mState != UNINITIALIZED) { 1521 PostReplyWithError(replyID, INVALID_OPERATION); 1522 break; 1523 } 1524 1525 mReplyID = replyID; 1526 setState(INITIALIZING); 1527 1528 AString name; 1529 CHECK(msg->findString("name", &name)); 1530 1531 int32_t nameIsType; 1532 int32_t encoder = false; 1533 CHECK(msg->findInt32("nameIsType", &nameIsType)); 1534 if (nameIsType) { 1535 CHECK(msg->findInt32("encoder", &encoder)); 1536 } 1537 1538 sp<AMessage> format = new AMessage; 1539 1540 if (nameIsType) { 1541 format->setString("mime", name.c_str()); 1542 format->setInt32("encoder", encoder); 1543 } else { 1544 format->setString("componentName", name.c_str()); 1545 } 1546 1547 mCodec->initiateAllocateComponent(format); 1548 break; 1549 } 1550 1551 case kWhatSetNotification: 1552 { 1553 sp<AMessage> notify; 1554 if (msg->findMessage("on-frame-rendered", ¬ify)) { 1555 mOnFrameRenderedNotification = notify; 1556 } 1557 break; 1558 } 1559 1560 case kWhatSetCallback: 1561 { 1562 sp<AReplyToken> replyID; 1563 CHECK(msg->senderAwaitsResponse(&replyID)); 1564 1565 if (mState == UNINITIALIZED 1566 || mState == INITIALIZING 1567 || isExecuting()) { 1568 // callback can't be set after codec is executing, 1569 // or before it's initialized (as the callback 1570 // will be cleared when it goes to INITIALIZED) 1571 PostReplyWithError(replyID, INVALID_OPERATION); 1572 break; 1573 } 1574 1575 sp<AMessage> callback; 1576 CHECK(msg->findMessage("callback", &callback)); 1577 1578 mCallback = callback; 1579 1580 if (mCallback != NULL) { 1581 ALOGI("MediaCodec will operate in async mode"); 1582 mFlags |= kFlagIsAsync; 1583 } else { 1584 mFlags &= ~kFlagIsAsync; 1585 } 1586 1587 sp<AMessage> response = new AMessage; 1588 response->postReply(replyID); 1589 break; 1590 } 1591 1592 case kWhatConfigure: 1593 { 1594 sp<AReplyToken> replyID; 1595 CHECK(msg->senderAwaitsResponse(&replyID)); 1596 1597 if (mState != INITIALIZED) { 1598 PostReplyWithError(replyID, INVALID_OPERATION); 1599 break; 1600 } 1601 1602 sp<RefBase> obj; 1603 CHECK(msg->findObject("surface", &obj)); 1604 1605 sp<AMessage> format; 1606 CHECK(msg->findMessage("format", &format)); 1607 1608 int32_t push; 1609 if (msg->findInt32("push-blank-buffers-on-shutdown", &push) && push != 0) { 1610 mFlags |= kFlagPushBlankBuffersOnShutdown; 1611 } 1612 1613 if (obj != NULL) { 1614 format->setObject("native-window", obj); 1615 status_t err = handleSetSurface(static_cast<Surface *>(obj.get())); 1616 if (err != OK) { 1617 PostReplyWithError(replyID, err); 1618 break; 1619 } 1620 } else { 1621 handleSetSurface(NULL); 1622 } 1623 1624 mReplyID = replyID; 1625 setState(CONFIGURING); 1626 1627 void *crypto; 1628 if (!msg->findPointer("crypto", &crypto)) { 1629 crypto = NULL; 1630 } 1631 1632 mCrypto = static_cast<ICrypto *>(crypto); 1633 1634 uint32_t flags; 1635 CHECK(msg->findInt32("flags", (int32_t *)&flags)); 1636 1637 if (flags & CONFIGURE_FLAG_ENCODE) { 1638 format->setInt32("encoder", true); 1639 mFlags |= kFlagIsEncoder; 1640 } 1641 1642 extractCSD(format); 1643 1644 mCodec->initiateConfigureComponent(format); 1645 break; 1646 } 1647 1648 case kWhatSetSurface: 1649 { 1650 sp<AReplyToken> replyID; 1651 CHECK(msg->senderAwaitsResponse(&replyID)); 1652 1653 status_t err = OK; 1654 sp<Surface> surface; 1655 1656 switch (mState) { 1657 case CONFIGURED: 1658 case STARTED: 1659 case FLUSHED: 1660 { 1661 sp<RefBase> obj; 1662 (void)msg->findObject("surface", &obj); 1663 sp<Surface> surface = static_cast<Surface *>(obj.get()); 1664 if (mSurface == NULL) { 1665 // do not support setting surface if it was not set 1666 err = INVALID_OPERATION; 1667 } else if (obj == NULL) { 1668 // do not support unsetting surface 1669 err = BAD_VALUE; 1670 } else { 1671 err = connectToSurface(surface); 1672 if (err == BAD_VALUE) { 1673 // assuming reconnecting to same surface 1674 // TODO: check if it is the same surface 1675 err = OK; 1676 } else { 1677 if (err == OK) { 1678 if (mFlags & kFlagUsesSoftwareRenderer) { 1679 if (mSoftRenderer != NULL 1680 && (mFlags & kFlagPushBlankBuffersOnShutdown)) { 1681 pushBlankBuffersToNativeWindow(mSurface.get()); 1682 } 1683 mSoftRenderer = new SoftwareRenderer(surface); 1684 // TODO: check if this was successful 1685 } else { 1686 err = mCodec->setSurface(surface); 1687 } 1688 } 1689 if (err == OK) { 1690 (void)disconnectFromSurface(); 1691 mSurface = surface; 1692 } 1693 } 1694 } 1695 break; 1696 } 1697 1698 default: 1699 err = INVALID_OPERATION; 1700 break; 1701 } 1702 1703 PostReplyWithError(replyID, err); 1704 break; 1705 } 1706 1707 case kWhatCreateInputSurface: 1708 case kWhatSetInputSurface: 1709 { 1710 sp<AReplyToken> replyID; 1711 CHECK(msg->senderAwaitsResponse(&replyID)); 1712 1713 // Must be configured, but can't have been started yet. 1714 if (mState != CONFIGURED) { 1715 PostReplyWithError(replyID, INVALID_OPERATION); 1716 break; 1717 } 1718 1719 mReplyID = replyID; 1720 if (msg->what() == kWhatCreateInputSurface) { 1721 mCodec->initiateCreateInputSurface(); 1722 } else { 1723 sp<RefBase> obj; 1724 CHECK(msg->findObject("input-surface", &obj)); 1725 1726 mCodec->initiateSetInputSurface( 1727 static_cast<PersistentSurface *>(obj.get())); 1728 } 1729 break; 1730 } 1731 case kWhatStart: 1732 { 1733 sp<AReplyToken> replyID; 1734 CHECK(msg->senderAwaitsResponse(&replyID)); 1735 1736 if (mState == FLUSHED) { 1737 setState(STARTED); 1738 if (mHavePendingInputBuffers) { 1739 onInputBufferAvailable(); 1740 mHavePendingInputBuffers = false; 1741 } 1742 mCodec->signalResume(); 1743 PostReplyWithError(replyID, OK); 1744 break; 1745 } else if (mState != CONFIGURED) { 1746 PostReplyWithError(replyID, INVALID_OPERATION); 1747 break; 1748 } 1749 1750 mReplyID = replyID; 1751 setState(STARTING); 1752 1753 mCodec->initiateStart(); 1754 break; 1755 } 1756 1757 case kWhatStop: 1758 case kWhatRelease: 1759 { 1760 State targetState = 1761 (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED; 1762 1763 sp<AReplyToken> replyID; 1764 CHECK(msg->senderAwaitsResponse(&replyID)); 1765 1766 // already stopped/released 1767 if (mState == UNINITIALIZED && mReleasedByResourceManager) { 1768 sp<AMessage> response = new AMessage; 1769 response->setInt32("err", OK); 1770 response->postReply(replyID); 1771 break; 1772 } 1773 1774 int32_t reclaimed = 0; 1775 msg->findInt32("reclaimed", &reclaimed); 1776 if (reclaimed) { 1777 mReleasedByResourceManager = true; 1778 } 1779 1780 if (!((mFlags & kFlagIsComponentAllocated) && targetState == UNINITIALIZED) // See 1 1781 && mState != INITIALIZED 1782 && mState != CONFIGURED && !isExecuting()) { 1783 // 1) Permit release to shut down the component if allocated. 1784 // 1785 // 2) We may be in "UNINITIALIZED" state already and 1786 // also shutdown the encoder/decoder without the 1787 // client being aware of this if media server died while 1788 // we were being stopped. The client would assume that 1789 // after stop() returned, it would be safe to call release() 1790 // and it should be in this case, no harm to allow a release() 1791 // if we're already uninitialized. 1792 sp<AMessage> response = new AMessage; 1793 // TODO: we shouldn't throw an exception for stop/release. Change this to wait until 1794 // the previous stop/release completes and then reply with OK. 1795 status_t err = mState == targetState ? OK : INVALID_OPERATION; 1796 response->setInt32("err", err); 1797 if (err == OK && targetState == UNINITIALIZED) { 1798 mComponentName.clear(); 1799 } 1800 response->postReply(replyID); 1801 break; 1802 } 1803 1804 if (mFlags & kFlagSawMediaServerDie) { 1805 // It's dead, Jim. Don't expect initiateShutdown to yield 1806 // any useful results now... 1807 setState(UNINITIALIZED); 1808 if (targetState == UNINITIALIZED) { 1809 mComponentName.clear(); 1810 } 1811 (new AMessage)->postReply(replyID); 1812 break; 1813 } 1814 1815 mReplyID = replyID; 1816 setState(msg->what() == kWhatStop ? STOPPING : RELEASING); 1817 1818 mCodec->initiateShutdown( 1819 msg->what() == kWhatStop /* keepComponentAllocated */); 1820 1821 returnBuffersToCodec(); 1822 1823 if (mSoftRenderer != NULL && (mFlags & kFlagPushBlankBuffersOnShutdown)) { 1824 pushBlankBuffersToNativeWindow(mSurface.get()); 1825 } 1826 break; 1827 } 1828 1829 case kWhatDequeueInputBuffer: 1830 { 1831 sp<AReplyToken> replyID; 1832 CHECK(msg->senderAwaitsResponse(&replyID)); 1833 1834 if (mFlags & kFlagIsAsync) { 1835 ALOGE("dequeueOutputBuffer can't be used in async mode"); 1836 PostReplyWithError(replyID, INVALID_OPERATION); 1837 break; 1838 } 1839 1840 if (mHaveInputSurface) { 1841 ALOGE("dequeueInputBuffer can't be used with input surface"); 1842 PostReplyWithError(replyID, INVALID_OPERATION); 1843 break; 1844 } 1845 1846 if (handleDequeueInputBuffer(replyID, true /* new request */)) { 1847 break; 1848 } 1849 1850 int64_t timeoutUs; 1851 CHECK(msg->findInt64("timeoutUs", &timeoutUs)); 1852 1853 if (timeoutUs == 0ll) { 1854 PostReplyWithError(replyID, -EAGAIN); 1855 break; 1856 } 1857 1858 mFlags |= kFlagDequeueInputPending; 1859 mDequeueInputReplyID = replyID; 1860 1861 if (timeoutUs > 0ll) { 1862 sp<AMessage> timeoutMsg = 1863 new AMessage(kWhatDequeueInputTimedOut, this); 1864 timeoutMsg->setInt32( 1865 "generation", ++mDequeueInputTimeoutGeneration); 1866 timeoutMsg->post(timeoutUs); 1867 } 1868 break; 1869 } 1870 1871 case kWhatDequeueInputTimedOut: 1872 { 1873 int32_t generation; 1874 CHECK(msg->findInt32("generation", &generation)); 1875 1876 if (generation != mDequeueInputTimeoutGeneration) { 1877 // Obsolete 1878 break; 1879 } 1880 1881 CHECK(mFlags & kFlagDequeueInputPending); 1882 1883 PostReplyWithError(mDequeueInputReplyID, -EAGAIN); 1884 1885 mFlags &= ~kFlagDequeueInputPending; 1886 mDequeueInputReplyID = 0; 1887 break; 1888 } 1889 1890 case kWhatQueueInputBuffer: 1891 { 1892 sp<AReplyToken> replyID; 1893 CHECK(msg->senderAwaitsResponse(&replyID)); 1894 1895 if (!isExecuting()) { 1896 PostReplyWithError(replyID, INVALID_OPERATION); 1897 break; 1898 } else if (mFlags & kFlagStickyError) { 1899 PostReplyWithError(replyID, getStickyError()); 1900 break; 1901 } 1902 1903 status_t err = onQueueInputBuffer(msg); 1904 1905 PostReplyWithError(replyID, err); 1906 break; 1907 } 1908 1909 case kWhatDequeueOutputBuffer: 1910 { 1911 sp<AReplyToken> replyID; 1912 CHECK(msg->senderAwaitsResponse(&replyID)); 1913 1914 if (mFlags & kFlagIsAsync) { 1915 ALOGE("dequeueOutputBuffer can't be used in async mode"); 1916 PostReplyWithError(replyID, INVALID_OPERATION); 1917 break; 1918 } 1919 1920 if (handleDequeueOutputBuffer(replyID, true /* new request */)) { 1921 break; 1922 } 1923 1924 int64_t timeoutUs; 1925 CHECK(msg->findInt64("timeoutUs", &timeoutUs)); 1926 1927 if (timeoutUs == 0ll) { 1928 PostReplyWithError(replyID, -EAGAIN); 1929 break; 1930 } 1931 1932 mFlags |= kFlagDequeueOutputPending; 1933 mDequeueOutputReplyID = replyID; 1934 1935 if (timeoutUs > 0ll) { 1936 sp<AMessage> timeoutMsg = 1937 new AMessage(kWhatDequeueOutputTimedOut, this); 1938 timeoutMsg->setInt32( 1939 "generation", ++mDequeueOutputTimeoutGeneration); 1940 timeoutMsg->post(timeoutUs); 1941 } 1942 break; 1943 } 1944 1945 case kWhatDequeueOutputTimedOut: 1946 { 1947 int32_t generation; 1948 CHECK(msg->findInt32("generation", &generation)); 1949 1950 if (generation != mDequeueOutputTimeoutGeneration) { 1951 // Obsolete 1952 break; 1953 } 1954 1955 CHECK(mFlags & kFlagDequeueOutputPending); 1956 1957 PostReplyWithError(mDequeueOutputReplyID, -EAGAIN); 1958 1959 mFlags &= ~kFlagDequeueOutputPending; 1960 mDequeueOutputReplyID = 0; 1961 break; 1962 } 1963 1964 case kWhatReleaseOutputBuffer: 1965 { 1966 sp<AReplyToken> replyID; 1967 CHECK(msg->senderAwaitsResponse(&replyID)); 1968 1969 if (!isExecuting()) { 1970 PostReplyWithError(replyID, INVALID_OPERATION); 1971 break; 1972 } else if (mFlags & kFlagStickyError) { 1973 PostReplyWithError(replyID, getStickyError()); 1974 break; 1975 } 1976 1977 status_t err = onReleaseOutputBuffer(msg); 1978 1979 PostReplyWithError(replyID, err); 1980 break; 1981 } 1982 1983 case kWhatSignalEndOfInputStream: 1984 { 1985 sp<AReplyToken> replyID; 1986 CHECK(msg->senderAwaitsResponse(&replyID)); 1987 1988 if (!isExecuting()) { 1989 PostReplyWithError(replyID, INVALID_OPERATION); 1990 break; 1991 } else if (mFlags & kFlagStickyError) { 1992 PostReplyWithError(replyID, getStickyError()); 1993 break; 1994 } 1995 1996 mReplyID = replyID; 1997 mCodec->signalEndOfInputStream(); 1998 break; 1999 } 2000 2001 case kWhatGetBuffers: 2002 { 2003 sp<AReplyToken> replyID; 2004 CHECK(msg->senderAwaitsResponse(&replyID)); 2005 // Unfortunately widevine legacy source requires knowing all of the 2006 // codec input buffers, so we have to provide them even in async mode. 2007 int32_t widevine = 0; 2008 msg->findInt32("widevine", &widevine); 2009 2010 if (!isExecuting() || ((mFlags & kFlagIsAsync) && !widevine)) { 2011 PostReplyWithError(replyID, INVALID_OPERATION); 2012 break; 2013 } else if (mFlags & kFlagStickyError) { 2014 PostReplyWithError(replyID, getStickyError()); 2015 break; 2016 } 2017 2018 int32_t portIndex; 2019 CHECK(msg->findInt32("portIndex", &portIndex)); 2020 2021 Vector<sp<ABuffer> > *dstBuffers; 2022 CHECK(msg->findPointer("buffers", (void **)&dstBuffers)); 2023 2024 dstBuffers->clear(); 2025 const Vector<BufferInfo> &srcBuffers = mPortBuffers[portIndex]; 2026 2027 for (size_t i = 0; i < srcBuffers.size(); ++i) { 2028 const BufferInfo &info = srcBuffers.itemAt(i); 2029 2030 dstBuffers->push_back( 2031 (portIndex == kPortIndexInput && mCrypto != NULL) 2032 ? info.mEncryptedData : info.mData); 2033 } 2034 2035 (new AMessage)->postReply(replyID); 2036 break; 2037 } 2038 2039 case kWhatFlush: 2040 { 2041 sp<AReplyToken> replyID; 2042 CHECK(msg->senderAwaitsResponse(&replyID)); 2043 2044 if (!isExecuting()) { 2045 PostReplyWithError(replyID, INVALID_OPERATION); 2046 break; 2047 } else if (mFlags & kFlagStickyError) { 2048 PostReplyWithError(replyID, getStickyError()); 2049 break; 2050 } 2051 2052 mReplyID = replyID; 2053 // TODO: skip flushing if already FLUSHED 2054 setState(FLUSHING); 2055 2056 mCodec->signalFlush(); 2057 returnBuffersToCodec(); 2058 break; 2059 } 2060 2061 case kWhatGetInputFormat: 2062 case kWhatGetOutputFormat: 2063 { 2064 sp<AMessage> format = 2065 (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat); 2066 2067 sp<AReplyToken> replyID; 2068 CHECK(msg->senderAwaitsResponse(&replyID)); 2069 2070 if ((mState != CONFIGURED && mState != STARTING && 2071 mState != STARTED && mState != FLUSHING && 2072 mState != FLUSHED) 2073 || format == NULL) { 2074 PostReplyWithError(replyID, INVALID_OPERATION); 2075 break; 2076 } else if (mFlags & kFlagStickyError) { 2077 PostReplyWithError(replyID, getStickyError()); 2078 break; 2079 } 2080 2081 sp<AMessage> response = new AMessage; 2082 response->setMessage("format", format); 2083 response->postReply(replyID); 2084 break; 2085 } 2086 2087 case kWhatRequestIDRFrame: 2088 { 2089 mCodec->signalRequestIDRFrame(); 2090 break; 2091 } 2092 2093 case kWhatRequestActivityNotification: 2094 { 2095 CHECK(mActivityNotify == NULL); 2096 CHECK(msg->findMessage("notify", &mActivityNotify)); 2097 2098 postActivityNotificationIfPossible(); 2099 break; 2100 } 2101 2102 case kWhatGetName: 2103 { 2104 sp<AReplyToken> replyID; 2105 CHECK(msg->senderAwaitsResponse(&replyID)); 2106 2107 if (mComponentName.empty()) { 2108 PostReplyWithError(replyID, INVALID_OPERATION); 2109 break; 2110 } 2111 2112 sp<AMessage> response = new AMessage; 2113 response->setString("name", mComponentName.c_str()); 2114 response->postReply(replyID); 2115 break; 2116 } 2117 2118 case kWhatSetParameters: 2119 { 2120 sp<AReplyToken> replyID; 2121 CHECK(msg->senderAwaitsResponse(&replyID)); 2122 2123 sp<AMessage> params; 2124 CHECK(msg->findMessage("params", ¶ms)); 2125 2126 status_t err = onSetParameters(params); 2127 2128 PostReplyWithError(replyID, err); 2129 break; 2130 } 2131 2132 default: 2133 TRESPASS(); 2134 } 2135} 2136 2137void MediaCodec::extractCSD(const sp<AMessage> &format) { 2138 mCSD.clear(); 2139 2140 size_t i = 0; 2141 for (;;) { 2142 sp<ABuffer> csd; 2143 if (!format->findBuffer(AStringPrintf("csd-%u", i).c_str(), &csd)) { 2144 break; 2145 } 2146 2147 mCSD.push_back(csd); 2148 ++i; 2149 } 2150 2151 ALOGV("Found %zu pieces of codec specific data.", mCSD.size()); 2152} 2153 2154status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) { 2155 CHECK(!mCSD.empty()); 2156 2157 const BufferInfo *info = 2158 &mPortBuffers[kPortIndexInput].itemAt(bufferIndex); 2159 2160 sp<ABuffer> csd = *mCSD.begin(); 2161 mCSD.erase(mCSD.begin()); 2162 2163 const sp<ABuffer> &codecInputData = 2164 (mCrypto != NULL) ? info->mEncryptedData : info->mData; 2165 2166 if (csd->size() > codecInputData->capacity()) { 2167 return -EINVAL; 2168 } 2169 2170 memcpy(codecInputData->data(), csd->data(), csd->size()); 2171 2172 AString errorDetailMsg; 2173 2174 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); 2175 msg->setSize("index", bufferIndex); 2176 msg->setSize("offset", 0); 2177 msg->setSize("size", csd->size()); 2178 msg->setInt64("timeUs", 0ll); 2179 msg->setInt32("flags", BUFFER_FLAG_CODECCONFIG); 2180 msg->setPointer("errorDetailMsg", &errorDetailMsg); 2181 2182 return onQueueInputBuffer(msg); 2183} 2184 2185void MediaCodec::setState(State newState) { 2186 if (newState == INITIALIZED || newState == UNINITIALIZED) { 2187 delete mSoftRenderer; 2188 mSoftRenderer = NULL; 2189 2190 mCrypto.clear(); 2191 handleSetSurface(NULL); 2192 2193 mInputFormat.clear(); 2194 mOutputFormat.clear(); 2195 mFlags &= ~kFlagOutputFormatChanged; 2196 mFlags &= ~kFlagOutputBuffersChanged; 2197 mFlags &= ~kFlagStickyError; 2198 mFlags &= ~kFlagIsEncoder; 2199 mFlags &= ~kFlagGatherCodecSpecificData; 2200 mFlags &= ~kFlagIsAsync; 2201 mStickyError = OK; 2202 2203 mActivityNotify.clear(); 2204 mCallback.clear(); 2205 } 2206 2207 if (newState == UNINITIALIZED) { 2208 // return any straggling buffers, e.g. if we got here on an error 2209 returnBuffersToCodec(); 2210 2211 // The component is gone, mediaserver's probably back up already 2212 // but should definitely be back up should we try to instantiate 2213 // another component.. and the cycle continues. 2214 mFlags &= ~kFlagSawMediaServerDie; 2215 } 2216 2217 mState = newState; 2218 2219 cancelPendingDequeueOperations(); 2220 2221 updateBatteryStat(); 2222} 2223 2224void MediaCodec::returnBuffersToCodec() { 2225 returnBuffersToCodecOnPort(kPortIndexInput); 2226 returnBuffersToCodecOnPort(kPortIndexOutput); 2227} 2228 2229void MediaCodec::returnBuffersToCodecOnPort(int32_t portIndex) { 2230 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); 2231 Mutex::Autolock al(mBufferLock); 2232 2233 Vector<BufferInfo> *buffers = &mPortBuffers[portIndex]; 2234 2235 for (size_t i = 0; i < buffers->size(); ++i) { 2236 BufferInfo *info = &buffers->editItemAt(i); 2237 2238 if (info->mNotify != NULL) { 2239 sp<AMessage> msg = info->mNotify; 2240 info->mNotify = NULL; 2241 info->mOwnedByClient = false; 2242 2243 if (portIndex == kPortIndexInput) { 2244 /* no error, just returning buffers */ 2245 msg->setInt32("err", OK); 2246 } 2247 msg->post(); 2248 } 2249 } 2250 2251 mAvailPortBuffers[portIndex].clear(); 2252} 2253 2254size_t MediaCodec::updateBuffers( 2255 int32_t portIndex, const sp<AMessage> &msg) { 2256 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); 2257 2258 uint32_t bufferID; 2259 CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID)); 2260 2261 Vector<BufferInfo> *buffers = &mPortBuffers[portIndex]; 2262 2263 for (size_t i = 0; i < buffers->size(); ++i) { 2264 BufferInfo *info = &buffers->editItemAt(i); 2265 2266 if (info->mBufferID == bufferID) { 2267 CHECK(info->mNotify == NULL); 2268 CHECK(msg->findMessage("reply", &info->mNotify)); 2269 2270 info->mFormat = 2271 (portIndex == kPortIndexInput) ? mInputFormat : mOutputFormat; 2272 mAvailPortBuffers[portIndex].push_back(i); 2273 2274 return i; 2275 } 2276 } 2277 2278 TRESPASS(); 2279 2280 return 0; 2281} 2282 2283status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) { 2284 size_t index; 2285 size_t offset; 2286 size_t size; 2287 int64_t timeUs; 2288 uint32_t flags; 2289 CHECK(msg->findSize("index", &index)); 2290 CHECK(msg->findSize("offset", &offset)); 2291 CHECK(msg->findInt64("timeUs", &timeUs)); 2292 CHECK(msg->findInt32("flags", (int32_t *)&flags)); 2293 2294 const CryptoPlugin::SubSample *subSamples; 2295 size_t numSubSamples; 2296 const uint8_t *key; 2297 const uint8_t *iv; 2298 CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted; 2299 2300 // We allow the simpler queueInputBuffer API to be used even in 2301 // secure mode, by fabricating a single unencrypted subSample. 2302 CryptoPlugin::SubSample ss; 2303 2304 if (msg->findSize("size", &size)) { 2305 if (mCrypto != NULL) { 2306 ss.mNumBytesOfClearData = size; 2307 ss.mNumBytesOfEncryptedData = 0; 2308 2309 subSamples = &ss; 2310 numSubSamples = 1; 2311 key = NULL; 2312 iv = NULL; 2313 } 2314 } else { 2315 if (mCrypto == NULL) { 2316 return -EINVAL; 2317 } 2318 2319 CHECK(msg->findPointer("subSamples", (void **)&subSamples)); 2320 CHECK(msg->findSize("numSubSamples", &numSubSamples)); 2321 CHECK(msg->findPointer("key", (void **)&key)); 2322 CHECK(msg->findPointer("iv", (void **)&iv)); 2323 2324 int32_t tmp; 2325 CHECK(msg->findInt32("mode", &tmp)); 2326 2327 mode = (CryptoPlugin::Mode)tmp; 2328 2329 size = 0; 2330 for (size_t i = 0; i < numSubSamples; ++i) { 2331 size += subSamples[i].mNumBytesOfClearData; 2332 size += subSamples[i].mNumBytesOfEncryptedData; 2333 } 2334 } 2335 2336 if (index >= mPortBuffers[kPortIndexInput].size()) { 2337 return -ERANGE; 2338 } 2339 2340 BufferInfo *info = &mPortBuffers[kPortIndexInput].editItemAt(index); 2341 2342 if (info->mNotify == NULL || !info->mOwnedByClient) { 2343 return -EACCES; 2344 } 2345 2346 if (offset + size > info->mData->capacity()) { 2347 return -EINVAL; 2348 } 2349 2350 sp<AMessage> reply = info->mNotify; 2351 info->mData->setRange(offset, size); 2352 info->mData->meta()->setInt64("timeUs", timeUs); 2353 2354 if (flags & BUFFER_FLAG_EOS) { 2355 info->mData->meta()->setInt32("eos", true); 2356 } 2357 2358 if (flags & BUFFER_FLAG_CODECCONFIG) { 2359 info->mData->meta()->setInt32("csd", true); 2360 } 2361 2362 if (mCrypto != NULL) { 2363 if (size > info->mEncryptedData->capacity()) { 2364 return -ERANGE; 2365 } 2366 2367 AString *errorDetailMsg; 2368 CHECK(msg->findPointer("errorDetailMsg", (void **)&errorDetailMsg)); 2369 2370 ssize_t result = mCrypto->decrypt( 2371 (mFlags & kFlagIsSecure) != 0, 2372 key, 2373 iv, 2374 mode, 2375 info->mSharedEncryptedBuffer, 2376 offset, 2377 subSamples, 2378 numSubSamples, 2379 info->mData->base(), 2380 errorDetailMsg); 2381 2382 if (result < 0) { 2383 return result; 2384 } 2385 2386 info->mData->setRange(0, result); 2387 } 2388 2389 // synchronization boundary for getBufferAndFormat 2390 { 2391 Mutex::Autolock al(mBufferLock); 2392 info->mOwnedByClient = false; 2393 } 2394 reply->setBuffer("buffer", info->mData); 2395 reply->post(); 2396 2397 info->mNotify = NULL; 2398 2399 return OK; 2400} 2401 2402//static 2403size_t MediaCodec::CreateFramesRenderedMessage( 2404 std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg) { 2405 size_t index = 0; 2406 2407 for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin(); 2408 it != done.cend(); ++it) { 2409 if (it->getRenderTimeNs() < 0) { 2410 continue; // dropped frame from tracking 2411 } 2412 msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs()); 2413 msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs()); 2414 ++index; 2415 } 2416 return index; 2417} 2418 2419status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) { 2420 size_t index; 2421 CHECK(msg->findSize("index", &index)); 2422 2423 int32_t render; 2424 if (!msg->findInt32("render", &render)) { 2425 render = 0; 2426 } 2427 2428 if (!isExecuting()) { 2429 return -EINVAL; 2430 } 2431 2432 if (index >= mPortBuffers[kPortIndexOutput].size()) { 2433 return -ERANGE; 2434 } 2435 2436 BufferInfo *info = &mPortBuffers[kPortIndexOutput].editItemAt(index); 2437 2438 if (info->mNotify == NULL || !info->mOwnedByClient) { 2439 return -EACCES; 2440 } 2441 2442 // synchronization boundary for getBufferAndFormat 2443 { 2444 Mutex::Autolock al(mBufferLock); 2445 info->mOwnedByClient = false; 2446 } 2447 2448 if (render && info->mData != NULL && info->mData->size() != 0) { 2449 info->mNotify->setInt32("render", true); 2450 2451 int64_t mediaTimeUs = -1; 2452 info->mData->meta()->findInt64("timeUs", &mediaTimeUs); 2453 2454 int64_t renderTimeNs = 0; 2455 if (!msg->findInt64("timestampNs", &renderTimeNs)) { 2456 // use media timestamp if client did not request a specific render timestamp 2457 ALOGV("using buffer PTS of %lld", (long long)mediaTimeUs); 2458 renderTimeNs = mediaTimeUs * 1000; 2459 } 2460 info->mNotify->setInt64("timestampNs", renderTimeNs); 2461 2462 if (mSoftRenderer != NULL) { 2463 std::list<FrameRenderTracker::Info> doneFrames = mSoftRenderer->render( 2464 info->mData->data(), info->mData->size(), 2465 mediaTimeUs, renderTimeNs, NULL, info->mFormat); 2466 2467 // if we are running, notify rendered frames 2468 if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) { 2469 sp<AMessage> notify = mOnFrameRenderedNotification->dup(); 2470 sp<AMessage> data = new AMessage; 2471 if (CreateFramesRenderedMessage(doneFrames, data)) { 2472 notify->setMessage("data", data); 2473 notify->post(); 2474 } 2475 } 2476 } 2477 } 2478 2479 info->mNotify->post(); 2480 info->mNotify = NULL; 2481 2482 return OK; 2483} 2484 2485ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) { 2486 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); 2487 2488 List<size_t> *availBuffers = &mAvailPortBuffers[portIndex]; 2489 2490 if (availBuffers->empty()) { 2491 return -EAGAIN; 2492 } 2493 2494 size_t index = *availBuffers->begin(); 2495 availBuffers->erase(availBuffers->begin()); 2496 2497 BufferInfo *info = &mPortBuffers[portIndex].editItemAt(index); 2498 CHECK(!info->mOwnedByClient); 2499 { 2500 Mutex::Autolock al(mBufferLock); 2501 info->mOwnedByClient = true; 2502 2503 // set image-data 2504 if (info->mFormat != NULL) { 2505 sp<ABuffer> imageData; 2506 if (info->mFormat->findBuffer("image-data", &imageData)) { 2507 info->mData->meta()->setBuffer("image-data", imageData); 2508 } 2509 int32_t left, top, right, bottom; 2510 if (info->mFormat->findRect("crop", &left, &top, &right, &bottom)) { 2511 info->mData->meta()->setRect("crop-rect", left, top, right, bottom); 2512 } 2513 } 2514 } 2515 2516 return index; 2517} 2518 2519status_t MediaCodec::connectToSurface(const sp<Surface> &surface) { 2520 status_t err = OK; 2521 if (surface != NULL) { 2522 err = native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA); 2523 if (err == BAD_VALUE) { 2524 ALOGI("native window already connected. Assuming no change of surface"); 2525 } else if (err != OK) { 2526 ALOGE("native_window_api_connect returned an error: %s (%d)", strerror(-err), err); 2527 } 2528 } 2529 return err; 2530} 2531 2532status_t MediaCodec::disconnectFromSurface() { 2533 status_t err = OK; 2534 if (mSurface != NULL) { 2535 err = native_window_api_disconnect(mSurface.get(), NATIVE_WINDOW_API_MEDIA); 2536 if (err != OK) { 2537 ALOGW("native_window_api_disconnect returned an error: %s (%d)", strerror(-err), err); 2538 } 2539 // assume disconnected even on error 2540 mSurface.clear(); 2541 } 2542 return err; 2543} 2544 2545status_t MediaCodec::handleSetSurface(const sp<Surface> &surface) { 2546 status_t err = OK; 2547 if (mSurface != NULL) { 2548 (void)disconnectFromSurface(); 2549 } 2550 if (surface != NULL) { 2551 err = connectToSurface(surface); 2552 if (err == OK) { 2553 mSurface = surface; 2554 } 2555 } 2556 return err; 2557} 2558 2559void MediaCodec::onInputBufferAvailable() { 2560 int32_t index; 2561 while ((index = dequeuePortBuffer(kPortIndexInput)) >= 0) { 2562 sp<AMessage> msg = mCallback->dup(); 2563 msg->setInt32("callbackID", CB_INPUT_AVAILABLE); 2564 msg->setInt32("index", index); 2565 msg->post(); 2566 } 2567} 2568 2569void MediaCodec::onOutputBufferAvailable() { 2570 int32_t index; 2571 while ((index = dequeuePortBuffer(kPortIndexOutput)) >= 0) { 2572 const sp<ABuffer> &buffer = 2573 mPortBuffers[kPortIndexOutput].itemAt(index).mData; 2574 sp<AMessage> msg = mCallback->dup(); 2575 msg->setInt32("callbackID", CB_OUTPUT_AVAILABLE); 2576 msg->setInt32("index", index); 2577 msg->setSize("offset", buffer->offset()); 2578 msg->setSize("size", buffer->size()); 2579 2580 int64_t timeUs; 2581 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 2582 2583 msg->setInt64("timeUs", timeUs); 2584 2585 int32_t omxFlags; 2586 CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags)); 2587 2588 uint32_t flags = 0; 2589 if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) { 2590 flags |= BUFFER_FLAG_SYNCFRAME; 2591 } 2592 if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) { 2593 flags |= BUFFER_FLAG_CODECCONFIG; 2594 } 2595 if (omxFlags & OMX_BUFFERFLAG_EOS) { 2596 flags |= BUFFER_FLAG_EOS; 2597 } 2598 2599 msg->setInt32("flags", flags); 2600 2601 msg->post(); 2602 } 2603} 2604 2605void MediaCodec::onError(status_t err, int32_t actionCode, const char *detail) { 2606 if (mCallback != NULL) { 2607 sp<AMessage> msg = mCallback->dup(); 2608 msg->setInt32("callbackID", CB_ERROR); 2609 msg->setInt32("err", err); 2610 msg->setInt32("actionCode", actionCode); 2611 2612 if (detail != NULL) { 2613 msg->setString("detail", detail); 2614 } 2615 2616 msg->post(); 2617 } 2618} 2619 2620void MediaCodec::onOutputFormatChanged() { 2621 if (mCallback != NULL) { 2622 sp<AMessage> msg = mCallback->dup(); 2623 msg->setInt32("callbackID", CB_OUTPUT_FORMAT_CHANGED); 2624 msg->setMessage("format", mOutputFormat); 2625 msg->post(); 2626 } 2627} 2628 2629 2630void MediaCodec::postActivityNotificationIfPossible() { 2631 if (mActivityNotify == NULL) { 2632 return; 2633 } 2634 2635 bool isErrorOrOutputChanged = 2636 (mFlags & (kFlagStickyError 2637 | kFlagOutputBuffersChanged 2638 | kFlagOutputFormatChanged)); 2639 2640 if (isErrorOrOutputChanged 2641 || !mAvailPortBuffers[kPortIndexInput].empty() 2642 || !mAvailPortBuffers[kPortIndexOutput].empty()) { 2643 mActivityNotify->setInt32("input-buffers", 2644 mAvailPortBuffers[kPortIndexInput].size()); 2645 2646 if (isErrorOrOutputChanged) { 2647 // we want consumer to dequeue as many times as it can 2648 mActivityNotify->setInt32("output-buffers", INT32_MAX); 2649 } else { 2650 mActivityNotify->setInt32("output-buffers", 2651 mAvailPortBuffers[kPortIndexOutput].size()); 2652 } 2653 mActivityNotify->post(); 2654 mActivityNotify.clear(); 2655 } 2656} 2657 2658status_t MediaCodec::setParameters(const sp<AMessage> ¶ms) { 2659 sp<AMessage> msg = new AMessage(kWhatSetParameters, this); 2660 msg->setMessage("params", params); 2661 2662 sp<AMessage> response; 2663 return PostAndAwaitResponse(msg, &response); 2664} 2665 2666status_t MediaCodec::onSetParameters(const sp<AMessage> ¶ms) { 2667 mCodec->signalSetParameters(params); 2668 2669 return OK; 2670} 2671 2672status_t MediaCodec::amendOutputFormatWithCodecSpecificData( 2673 const sp<ABuffer> &buffer) { 2674 AString mime; 2675 CHECK(mOutputFormat->findString("mime", &mime)); 2676 2677 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) { 2678 // Codec specific data should be SPS and PPS in a single buffer, 2679 // each prefixed by a startcode (0x00 0x00 0x00 0x01). 2680 // We separate the two and put them into the output format 2681 // under the keys "csd-0" and "csd-1". 2682 2683 unsigned csdIndex = 0; 2684 2685 const uint8_t *data = buffer->data(); 2686 size_t size = buffer->size(); 2687 2688 const uint8_t *nalStart; 2689 size_t nalSize; 2690 while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) { 2691 sp<ABuffer> csd = new ABuffer(nalSize + 4); 2692 memcpy(csd->data(), "\x00\x00\x00\x01", 4); 2693 memcpy(csd->data() + 4, nalStart, nalSize); 2694 2695 mOutputFormat->setBuffer( 2696 AStringPrintf("csd-%u", csdIndex).c_str(), csd); 2697 2698 ++csdIndex; 2699 } 2700 2701 if (csdIndex != 2) { 2702 return ERROR_MALFORMED; 2703 } 2704 } else { 2705 // For everything else we just stash the codec specific data into 2706 // the output format as a single piece of csd under "csd-0". 2707 mOutputFormat->setBuffer("csd-0", buffer); 2708 } 2709 2710 return OK; 2711} 2712 2713void MediaCodec::updateBatteryStat() { 2714 if (mState == CONFIGURED && !mBatteryStatNotified) { 2715 BatteryNotifier& notifier(BatteryNotifier::getInstance()); 2716 2717 if (mIsVideo) { 2718 notifier.noteStartVideo(); 2719 } else { 2720 notifier.noteStartAudio(); 2721 } 2722 2723 mBatteryStatNotified = true; 2724 } else if (mState == UNINITIALIZED && mBatteryStatNotified) { 2725 BatteryNotifier& notifier(BatteryNotifier::getInstance()); 2726 2727 if (mIsVideo) { 2728 notifier.noteStopVideo(); 2729 } else { 2730 notifier.noteStopAudio(); 2731 } 2732 2733 mBatteryStatNotified = false; 2734 } 2735} 2736 2737} // namespace android 2738