MediaCodec.cpp revision 0362655ca9494052f348f83dabecf9ea27003976
1/* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "MediaCodec" 19#include <inttypes.h> 20 21#include "include/avc_utils.h" 22#include "include/SoftwareRenderer.h" 23 24#include <binder/IMemory.h> 25#include <binder/IPCThreadState.h> 26#include <binder/IServiceManager.h> 27#include <binder/MemoryDealer.h> 28#include <gui/BufferQueue.h> 29#include <gui/Surface.h> 30#include <media/ICrypto.h> 31#include <media/IOMX.h> 32#include <media/IResourceManagerService.h> 33#include <media/stagefright/foundation/ABuffer.h> 34#include <media/stagefright/foundation/ADebug.h> 35#include <media/stagefright/foundation/AMessage.h> 36#include <media/stagefright/foundation/AString.h> 37#include <media/stagefright/foundation/hexdump.h> 38#include <media/stagefright/ACodec.h> 39#include <media/stagefright/BufferProducerWrapper.h> 40#include <media/stagefright/MediaCodec.h> 41#include <media/stagefright/MediaCodecList.h> 42#include <media/stagefright/MediaDefs.h> 43#include <media/stagefright/MediaErrors.h> 44#include <media/stagefright/MediaFilter.h> 45#include <media/stagefright/MetaData.h> 46#include <media/stagefright/OMXClient.h> 47#include <media/stagefright/OMXCodec.h> 48#include <media/stagefright/PersistentSurface.h> 49#include <media/stagefright/SurfaceUtils.h> 50#include <mediautils/BatteryNotifier.h> 51#include <private/android_filesystem_config.h> 52#include <utils/Log.h> 53#include <utils/Singleton.h> 54 55namespace android { 56 57static int64_t getId(sp<IResourceManagerClient> client) { 58 return (int64_t) client.get(); 59} 60 61static bool isResourceError(status_t err) { 62 return (err == NO_MEMORY); 63} 64 65static const int kMaxRetry = 2; 66 67struct ResourceManagerClient : public BnResourceManagerClient { 68 ResourceManagerClient(MediaCodec* codec) : mMediaCodec(codec) {} 69 70 virtual bool reclaimResource() { 71 sp<MediaCodec> codec = mMediaCodec.promote(); 72 if (codec == NULL) { 73 // codec is already gone. 74 return true; 75 } 76 status_t err = codec->reclaim(); 77 if (err != OK) { 78 ALOGW("ResourceManagerClient failed to release codec with err %d", err); 79 } 80 return (err == OK); 81 } 82 83 virtual String8 getName() { 84 String8 ret; 85 sp<MediaCodec> codec = mMediaCodec.promote(); 86 if (codec == NULL) { 87 // codec is already gone. 88 return ret; 89 } 90 91 AString name; 92 if (codec->getName(&name) == OK) { 93 ret.setTo(name.c_str()); 94 } 95 return ret; 96 } 97 98protected: 99 virtual ~ResourceManagerClient() {} 100 101private: 102 wp<MediaCodec> mMediaCodec; 103 104 DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient); 105}; 106 107MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy(pid_t pid) 108 : mPid(pid) { 109 if (mPid == MediaCodec::kNoPid) { 110 mPid = IPCThreadState::self()->getCallingPid(); 111 } 112} 113 114MediaCodec::ResourceManagerServiceProxy::~ResourceManagerServiceProxy() { 115 if (mService != NULL) { 116 IInterface::asBinder(mService)->unlinkToDeath(this); 117 } 118} 119 120void MediaCodec::ResourceManagerServiceProxy::init() { 121 sp<IServiceManager> sm = defaultServiceManager(); 122 sp<IBinder> binder = sm->getService(String16("media.resource_manager")); 123 mService = interface_cast<IResourceManagerService>(binder); 124 if (mService == NULL) { 125 ALOGE("Failed to get ResourceManagerService"); 126 return; 127 } 128 IInterface::asBinder(mService)->linkToDeath(this); 129} 130 131void MediaCodec::ResourceManagerServiceProxy::binderDied(const wp<IBinder>& /*who*/) { 132 ALOGW("ResourceManagerService died."); 133 Mutex::Autolock _l(mLock); 134 mService.clear(); 135} 136 137void MediaCodec::ResourceManagerServiceProxy::addResource( 138 int64_t clientId, 139 const sp<IResourceManagerClient> client, 140 const Vector<MediaResource> &resources) { 141 Mutex::Autolock _l(mLock); 142 if (mService == NULL) { 143 return; 144 } 145 mService->addResource(mPid, clientId, client, resources); 146} 147 148void MediaCodec::ResourceManagerServiceProxy::removeResource(int64_t clientId) { 149 Mutex::Autolock _l(mLock); 150 if (mService == NULL) { 151 return; 152 } 153 mService->removeResource(mPid, clientId); 154} 155 156bool MediaCodec::ResourceManagerServiceProxy::reclaimResource( 157 const Vector<MediaResource> &resources) { 158 Mutex::Autolock _l(mLock); 159 if (mService == NULL) { 160 return false; 161 } 162 return mService->reclaimResource(mPid, resources); 163} 164 165// static 166sp<MediaCodec> MediaCodec::CreateByType( 167 const sp<ALooper> &looper, const char *mime, bool encoder, status_t *err, pid_t pid) { 168 sp<MediaCodec> codec = new MediaCodec(looper, pid); 169 170 const status_t ret = codec->init(mime, true /* nameIsType */, encoder); 171 if (err != NULL) { 172 *err = ret; 173 } 174 return ret == OK ? codec : NULL; // NULL deallocates codec. 175} 176 177// static 178sp<MediaCodec> MediaCodec::CreateByComponentName( 179 const sp<ALooper> &looper, const char *name, status_t *err, pid_t pid) { 180 sp<MediaCodec> codec = new MediaCodec(looper, pid); 181 182 const status_t ret = codec->init(name, false /* nameIsType */, false /* encoder */); 183 if (err != NULL) { 184 *err = ret; 185 } 186 return ret == OK ? codec : NULL; // NULL deallocates codec. 187} 188 189// static 190sp<PersistentSurface> MediaCodec::CreatePersistentInputSurface() { 191 OMXClient client; 192 CHECK_EQ(client.connect(), (status_t)OK); 193 sp<IOMX> omx = client.interface(); 194 195 const sp<IMediaCodecList> mediaCodecList = MediaCodecList::getInstance(); 196 if (mediaCodecList == NULL) { 197 ALOGE("Failed to obtain MediaCodecList!"); 198 return NULL; // if called from Java should raise IOException 199 } 200 201 AString tmp; 202 sp<AMessage> globalSettings = mediaCodecList->getGlobalSettings(); 203 if (globalSettings == NULL || !globalSettings->findString( 204 kMaxEncoderInputBuffers, &tmp)) { 205 ALOGE("Failed to get encoder input buffer count!"); 206 return NULL; 207 } 208 209 int32_t bufferCount = strtol(tmp.c_str(), NULL, 10); 210 if (bufferCount <= 0 211 || bufferCount > BufferQueue::MAX_MAX_ACQUIRED_BUFFERS) { 212 ALOGE("Encoder input buffer count is invalid!"); 213 return NULL; 214 } 215 216 sp<IGraphicBufferProducer> bufferProducer; 217 sp<IGraphicBufferConsumer> bufferConsumer; 218 219 status_t err = omx->createPersistentInputSurface( 220 &bufferProducer, &bufferConsumer); 221 222 if (err != OK) { 223 ALOGE("Failed to create persistent input surface."); 224 return NULL; 225 } 226 227 err = bufferConsumer->setMaxAcquiredBufferCount(bufferCount); 228 229 if (err != NO_ERROR) { 230 ALOGE("Unable to set BQ max acquired buffer count to %u: %d", 231 bufferCount, err); 232 return NULL; 233 } 234 235 return new PersistentSurface(bufferProducer, bufferConsumer); 236} 237 238MediaCodec::MediaCodec(const sp<ALooper> &looper, pid_t pid) 239 : mState(UNINITIALIZED), 240 mReleasedByResourceManager(false), 241 mLooper(looper), 242 mCodec(NULL), 243 mReplyID(0), 244 mFlags(0), 245 mStickyError(OK), 246 mSoftRenderer(NULL), 247 mResourceManagerClient(new ResourceManagerClient(this)), 248 mResourceManagerService(new ResourceManagerServiceProxy(pid)), 249 mBatteryStatNotified(false), 250 mIsVideo(false), 251 mVideoWidth(0), 252 mVideoHeight(0), 253 mRotationDegrees(0), 254 mDequeueInputTimeoutGeneration(0), 255 mDequeueInputReplyID(0), 256 mDequeueOutputTimeoutGeneration(0), 257 mDequeueOutputReplyID(0), 258 mHaveInputSurface(false), 259 mHavePendingInputBuffers(false) { 260} 261 262MediaCodec::~MediaCodec() { 263 CHECK_EQ(mState, UNINITIALIZED); 264 mResourceManagerService->removeResource(getId(mResourceManagerClient)); 265} 266 267// static 268status_t MediaCodec::PostAndAwaitResponse( 269 const sp<AMessage> &msg, sp<AMessage> *response) { 270 status_t err = msg->postAndAwaitResponse(response); 271 272 if (err != OK) { 273 return err; 274 } 275 276 if (!(*response)->findInt32("err", &err)) { 277 err = OK; 278 } 279 280 return err; 281} 282 283void MediaCodec::PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err) { 284 int32_t finalErr = err; 285 if (mReleasedByResourceManager) { 286 // override the err code if MediaCodec has been released by ResourceManager. 287 finalErr = DEAD_OBJECT; 288 } 289 290 sp<AMessage> response = new AMessage; 291 response->setInt32("err", finalErr); 292 response->postReply(replyID); 293} 294 295status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) { 296 mResourceManagerService->init(); 297 298 // save init parameters for reset 299 mInitName = name; 300 mInitNameIsType = nameIsType; 301 mInitIsEncoder = encoder; 302 303 // Current video decoders do not return from OMX_FillThisBuffer 304 // quickly, violating the OpenMAX specs, until that is remedied 305 // we need to invest in an extra looper to free the main event 306 // queue. 307 308 if (nameIsType || !strncasecmp(name.c_str(), "omx.", 4)) { 309 mCodec = new ACodec; 310 } else if (!nameIsType 311 && !strncasecmp(name.c_str(), "android.filter.", 15)) { 312 mCodec = new MediaFilter; 313 } else { 314 return NAME_NOT_FOUND; 315 } 316 317 bool secureCodec = false; 318 if (nameIsType && !strncasecmp(name.c_str(), "video/", 6)) { 319 mIsVideo = true; 320 } else { 321 AString tmp = name; 322 if (tmp.endsWith(".secure")) { 323 secureCodec = true; 324 tmp.erase(tmp.size() - 7, 7); 325 } 326 const sp<IMediaCodecList> mcl = MediaCodecList::getInstance(); 327 if (mcl == NULL) { 328 mCodec = NULL; // remove the codec. 329 return NO_INIT; // if called from Java should raise IOException 330 } 331 ssize_t codecIdx = mcl->findCodecByName(tmp.c_str()); 332 if (codecIdx >= 0) { 333 const sp<MediaCodecInfo> info = mcl->getCodecInfo(codecIdx); 334 Vector<AString> mimes; 335 info->getSupportedMimes(&mimes); 336 for (size_t i = 0; i < mimes.size(); i++) { 337 if (mimes[i].startsWith("video/")) { 338 mIsVideo = true; 339 break; 340 } 341 } 342 } 343 } 344 345 if (mIsVideo) { 346 // video codec needs dedicated looper 347 if (mCodecLooper == NULL) { 348 mCodecLooper = new ALooper; 349 mCodecLooper->setName("CodecLooper"); 350 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 351 } 352 353 mCodecLooper->registerHandler(mCodec); 354 } else { 355 mLooper->registerHandler(mCodec); 356 } 357 358 mLooper->registerHandler(this); 359 360 mCodec->setNotificationMessage(new AMessage(kWhatCodecNotify, this)); 361 362 sp<AMessage> msg = new AMessage(kWhatInit, this); 363 msg->setString("name", name); 364 msg->setInt32("nameIsType", nameIsType); 365 366 if (nameIsType) { 367 msg->setInt32("encoder", encoder); 368 } 369 370 status_t err; 371 Vector<MediaResource> resources; 372 const char *type = secureCodec ? kResourceSecureCodec : kResourceNonSecureCodec; 373 const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec; 374 resources.push_back(MediaResource(String8(type), String8(subtype), 1)); 375 for (int i = 0; i <= kMaxRetry; ++i) { 376 if (i > 0) { 377 // Don't try to reclaim resource for the first time. 378 if (!mResourceManagerService->reclaimResource(resources)) { 379 break; 380 } 381 } 382 383 sp<AMessage> response; 384 err = PostAndAwaitResponse(msg, &response); 385 if (!isResourceError(err)) { 386 break; 387 } 388 } 389 return err; 390} 391 392status_t MediaCodec::setCallback(const sp<AMessage> &callback) { 393 sp<AMessage> msg = new AMessage(kWhatSetCallback, this); 394 msg->setMessage("callback", callback); 395 396 sp<AMessage> response; 397 return PostAndAwaitResponse(msg, &response); 398} 399 400status_t MediaCodec::setOnFrameRenderedNotification(const sp<AMessage> ¬ify) { 401 sp<AMessage> msg = new AMessage(kWhatSetNotification, this); 402 msg->setMessage("on-frame-rendered", notify); 403 return msg->post(); 404} 405 406status_t MediaCodec::configure( 407 const sp<AMessage> &format, 408 const sp<Surface> &surface, 409 const sp<ICrypto> &crypto, 410 uint32_t flags) { 411 sp<AMessage> msg = new AMessage(kWhatConfigure, this); 412 413 if (mIsVideo) { 414 format->findInt32("width", &mVideoWidth); 415 format->findInt32("height", &mVideoHeight); 416 if (!format->findInt32("rotation-degrees", &mRotationDegrees)) { 417 mRotationDegrees = 0; 418 } 419 } 420 421 msg->setMessage("format", format); 422 msg->setInt32("flags", flags); 423 msg->setObject("surface", surface); 424 425 if (crypto != NULL) { 426 msg->setPointer("crypto", crypto.get()); 427 } 428 429 // save msg for reset 430 mConfigureMsg = msg; 431 432 status_t err; 433 Vector<MediaResource> resources; 434 const char *type = (mFlags & kFlagIsSecure) ? 435 kResourceSecureCodec : kResourceNonSecureCodec; 436 const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec; 437 resources.push_back(MediaResource(String8(type), String8(subtype), 1)); 438 // Don't know the buffer size at this point, but it's fine to use 1 because 439 // the reclaimResource call doesn't consider the requester's buffer size for now. 440 resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1)); 441 for (int i = 0; i <= kMaxRetry; ++i) { 442 if (i > 0) { 443 // Don't try to reclaim resource for the first time. 444 if (!mResourceManagerService->reclaimResource(resources)) { 445 break; 446 } 447 } 448 449 sp<AMessage> response; 450 err = PostAndAwaitResponse(msg, &response); 451 if (err != OK && err != INVALID_OPERATION) { 452 // MediaCodec now set state to UNINITIALIZED upon any fatal error. 453 // To maintain backward-compatibility, do a reset() to put codec 454 // back into INITIALIZED state. 455 // But don't reset if the err is INVALID_OPERATION, which means 456 // the configure failure is due to wrong state. 457 458 ALOGE("configure failed with err 0x%08x, resetting...", err); 459 reset(); 460 } 461 if (!isResourceError(err)) { 462 break; 463 } 464 } 465 return err; 466} 467 468status_t MediaCodec::setInputSurface( 469 const sp<PersistentSurface> &surface) { 470 sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this); 471 msg->setObject("input-surface", surface.get()); 472 473 sp<AMessage> response; 474 return PostAndAwaitResponse(msg, &response); 475} 476 477status_t MediaCodec::setSurface(const sp<Surface> &surface) { 478 sp<AMessage> msg = new AMessage(kWhatSetSurface, this); 479 msg->setObject("surface", surface); 480 481 sp<AMessage> response; 482 return PostAndAwaitResponse(msg, &response); 483} 484 485status_t MediaCodec::createInputSurface( 486 sp<IGraphicBufferProducer>* bufferProducer) { 487 sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, this); 488 489 sp<AMessage> response; 490 status_t err = PostAndAwaitResponse(msg, &response); 491 if (err == NO_ERROR) { 492 // unwrap the sp<IGraphicBufferProducer> 493 sp<RefBase> obj; 494 bool found = response->findObject("input-surface", &obj); 495 CHECK(found); 496 sp<BufferProducerWrapper> wrapper( 497 static_cast<BufferProducerWrapper*>(obj.get())); 498 *bufferProducer = wrapper->getBufferProducer(); 499 } else { 500 ALOGW("createInputSurface failed, err=%d", err); 501 } 502 return err; 503} 504 505uint64_t MediaCodec::getGraphicBufferSize() { 506 if (!mIsVideo) { 507 return 0; 508 } 509 510 uint64_t size = 0; 511 size_t portNum = sizeof(mPortBuffers) / sizeof((mPortBuffers)[0]); 512 for (size_t i = 0; i < portNum; ++i) { 513 // TODO: this is just an estimation, we should get the real buffer size from ACodec. 514 size += mPortBuffers[i].size() * mVideoWidth * mVideoHeight * 3 / 2; 515 } 516 return size; 517} 518 519void MediaCodec::addResource(const String8 &type, const String8 &subtype, uint64_t value) { 520 Vector<MediaResource> resources; 521 resources.push_back(MediaResource(type, subtype, value)); 522 mResourceManagerService->addResource( 523 getId(mResourceManagerClient), mResourceManagerClient, resources); 524} 525 526status_t MediaCodec::start() { 527 sp<AMessage> msg = new AMessage(kWhatStart, this); 528 529 status_t err; 530 Vector<MediaResource> resources; 531 const char *type = (mFlags & kFlagIsSecure) ? 532 kResourceSecureCodec : kResourceNonSecureCodec; 533 const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec; 534 resources.push_back(MediaResource(String8(type), String8(subtype), 1)); 535 // Don't know the buffer size at this point, but it's fine to use 1 because 536 // the reclaimResource call doesn't consider the requester's buffer size for now. 537 resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1)); 538 for (int i = 0; i <= kMaxRetry; ++i) { 539 if (i > 0) { 540 // Don't try to reclaim resource for the first time. 541 if (!mResourceManagerService->reclaimResource(resources)) { 542 break; 543 } 544 // Recover codec from previous error before retry start. 545 err = reset(); 546 if (err != OK) { 547 ALOGE("retrying start: failed to reset codec"); 548 break; 549 } 550 sp<AMessage> response; 551 err = PostAndAwaitResponse(mConfigureMsg, &response); 552 if (err != OK) { 553 ALOGE("retrying start: failed to configure codec"); 554 break; 555 } 556 } 557 558 sp<AMessage> response; 559 err = PostAndAwaitResponse(msg, &response); 560 if (!isResourceError(err)) { 561 break; 562 } 563 } 564 return err; 565} 566 567status_t MediaCodec::stop() { 568 sp<AMessage> msg = new AMessage(kWhatStop, this); 569 570 sp<AMessage> response; 571 return PostAndAwaitResponse(msg, &response); 572} 573 574status_t MediaCodec::reclaim() { 575 sp<AMessage> msg = new AMessage(kWhatRelease, this); 576 msg->setInt32("reclaimed", 1); 577 578 sp<AMessage> response; 579 return PostAndAwaitResponse(msg, &response); 580} 581 582status_t MediaCodec::release() { 583 sp<AMessage> msg = new AMessage(kWhatRelease, this); 584 585 sp<AMessage> response; 586 return PostAndAwaitResponse(msg, &response); 587} 588 589status_t MediaCodec::reset() { 590 /* When external-facing MediaCodec object is created, 591 it is already initialized. Thus, reset is essentially 592 release() followed by init(), plus clearing the state */ 593 594 status_t err = release(); 595 596 // unregister handlers 597 if (mCodec != NULL) { 598 if (mCodecLooper != NULL) { 599 mCodecLooper->unregisterHandler(mCodec->id()); 600 } else { 601 mLooper->unregisterHandler(mCodec->id()); 602 } 603 mCodec = NULL; 604 } 605 mLooper->unregisterHandler(id()); 606 607 mFlags = 0; // clear all flags 608 mStickyError = OK; 609 610 // reset state not reset by setState(UNINITIALIZED) 611 mReplyID = 0; 612 mDequeueInputReplyID = 0; 613 mDequeueOutputReplyID = 0; 614 mDequeueInputTimeoutGeneration = 0; 615 mDequeueOutputTimeoutGeneration = 0; 616 mHaveInputSurface = false; 617 618 if (err == OK) { 619 err = init(mInitName, mInitNameIsType, mInitIsEncoder); 620 } 621 return err; 622} 623 624status_t MediaCodec::queueInputBuffer( 625 size_t index, 626 size_t offset, 627 size_t size, 628 int64_t presentationTimeUs, 629 uint32_t flags, 630 AString *errorDetailMsg) { 631 if (errorDetailMsg != NULL) { 632 errorDetailMsg->clear(); 633 } 634 635 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); 636 msg->setSize("index", index); 637 msg->setSize("offset", offset); 638 msg->setSize("size", size); 639 msg->setInt64("timeUs", presentationTimeUs); 640 msg->setInt32("flags", flags); 641 msg->setPointer("errorDetailMsg", errorDetailMsg); 642 643 sp<AMessage> response; 644 return PostAndAwaitResponse(msg, &response); 645} 646 647status_t MediaCodec::queueSecureInputBuffer( 648 size_t index, 649 size_t offset, 650 const CryptoPlugin::SubSample *subSamples, 651 size_t numSubSamples, 652 const uint8_t key[16], 653 const uint8_t iv[16], 654 CryptoPlugin::Mode mode, 655 int64_t presentationTimeUs, 656 uint32_t flags, 657 AString *errorDetailMsg) { 658 if (errorDetailMsg != NULL) { 659 errorDetailMsg->clear(); 660 } 661 662 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); 663 msg->setSize("index", index); 664 msg->setSize("offset", offset); 665 msg->setPointer("subSamples", (void *)subSamples); 666 msg->setSize("numSubSamples", numSubSamples); 667 msg->setPointer("key", (void *)key); 668 msg->setPointer("iv", (void *)iv); 669 msg->setInt32("mode", mode); 670 msg->setInt64("timeUs", presentationTimeUs); 671 msg->setInt32("flags", flags); 672 msg->setPointer("errorDetailMsg", errorDetailMsg); 673 674 sp<AMessage> response; 675 status_t err = PostAndAwaitResponse(msg, &response); 676 677 return err; 678} 679 680status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) { 681 sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, this); 682 msg->setInt64("timeoutUs", timeoutUs); 683 684 sp<AMessage> response; 685 status_t err; 686 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 687 return err; 688 } 689 690 CHECK(response->findSize("index", index)); 691 692 return OK; 693} 694 695status_t MediaCodec::dequeueOutputBuffer( 696 size_t *index, 697 size_t *offset, 698 size_t *size, 699 int64_t *presentationTimeUs, 700 uint32_t *flags, 701 int64_t timeoutUs) { 702 sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, this); 703 msg->setInt64("timeoutUs", timeoutUs); 704 705 sp<AMessage> response; 706 status_t err; 707 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 708 return err; 709 } 710 711 CHECK(response->findSize("index", index)); 712 CHECK(response->findSize("offset", offset)); 713 CHECK(response->findSize("size", size)); 714 CHECK(response->findInt64("timeUs", presentationTimeUs)); 715 CHECK(response->findInt32("flags", (int32_t *)flags)); 716 717 return OK; 718} 719 720status_t MediaCodec::renderOutputBufferAndRelease(size_t index) { 721 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); 722 msg->setSize("index", index); 723 msg->setInt32("render", true); 724 725 sp<AMessage> response; 726 return PostAndAwaitResponse(msg, &response); 727} 728 729status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) { 730 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); 731 msg->setSize("index", index); 732 msg->setInt32("render", true); 733 msg->setInt64("timestampNs", timestampNs); 734 735 sp<AMessage> response; 736 return PostAndAwaitResponse(msg, &response); 737} 738 739status_t MediaCodec::releaseOutputBuffer(size_t index) { 740 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); 741 msg->setSize("index", index); 742 743 sp<AMessage> response; 744 return PostAndAwaitResponse(msg, &response); 745} 746 747status_t MediaCodec::signalEndOfInputStream() { 748 sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, this); 749 750 sp<AMessage> response; 751 return PostAndAwaitResponse(msg, &response); 752} 753 754status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const { 755 sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, this); 756 757 sp<AMessage> response; 758 status_t err; 759 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 760 return err; 761 } 762 763 CHECK(response->findMessage("format", format)); 764 765 return OK; 766} 767 768status_t MediaCodec::getInputFormat(sp<AMessage> *format) const { 769 sp<AMessage> msg = new AMessage(kWhatGetInputFormat, this); 770 771 sp<AMessage> response; 772 status_t err; 773 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 774 return err; 775 } 776 777 CHECK(response->findMessage("format", format)); 778 779 return OK; 780} 781 782status_t MediaCodec::getName(AString *name) const { 783 sp<AMessage> msg = new AMessage(kWhatGetName, this); 784 785 sp<AMessage> response; 786 status_t err; 787 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 788 return err; 789 } 790 791 CHECK(response->findString("name", name)); 792 793 return OK; 794} 795 796status_t MediaCodec::getWidevineLegacyBuffers(Vector<sp<ABuffer> > *buffers) const { 797 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this); 798 msg->setInt32("portIndex", kPortIndexInput); 799 msg->setPointer("buffers", buffers); 800 msg->setInt32("widevine", true); 801 802 sp<AMessage> response; 803 return PostAndAwaitResponse(msg, &response); 804} 805 806status_t MediaCodec::getInputBuffers(Vector<sp<ABuffer> > *buffers) const { 807 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this); 808 msg->setInt32("portIndex", kPortIndexInput); 809 msg->setPointer("buffers", buffers); 810 811 sp<AMessage> response; 812 return PostAndAwaitResponse(msg, &response); 813} 814 815status_t MediaCodec::getOutputBuffers(Vector<sp<ABuffer> > *buffers) const { 816 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this); 817 msg->setInt32("portIndex", kPortIndexOutput); 818 msg->setPointer("buffers", buffers); 819 820 sp<AMessage> response; 821 return PostAndAwaitResponse(msg, &response); 822} 823 824status_t MediaCodec::getOutputBuffer(size_t index, sp<ABuffer> *buffer) { 825 sp<AMessage> format; 826 return getBufferAndFormat(kPortIndexOutput, index, buffer, &format); 827} 828 829status_t MediaCodec::getOutputFormat(size_t index, sp<AMessage> *format) { 830 sp<ABuffer> buffer; 831 return getBufferAndFormat(kPortIndexOutput, index, &buffer, format); 832} 833 834status_t MediaCodec::getInputBuffer(size_t index, sp<ABuffer> *buffer) { 835 sp<AMessage> format; 836 return getBufferAndFormat(kPortIndexInput, index, buffer, &format); 837} 838 839bool MediaCodec::isExecuting() const { 840 return mState == STARTED || mState == FLUSHED; 841} 842 843status_t MediaCodec::getBufferAndFormat( 844 size_t portIndex, size_t index, 845 sp<ABuffer> *buffer, sp<AMessage> *format) { 846 // use mutex instead of a context switch 847 if (mReleasedByResourceManager) { 848 ALOGE("getBufferAndFormat - resource already released"); 849 return DEAD_OBJECT; 850 } 851 852 if (buffer == NULL) { 853 ALOGE("getBufferAndFormat - null ABuffer"); 854 return INVALID_OPERATION; 855 } 856 857 if (format == NULL) { 858 ALOGE("getBufferAndFormat - null AMessage"); 859 return INVALID_OPERATION; 860 } 861 862 buffer->clear(); 863 format->clear(); 864 865 if (!isExecuting()) { 866 ALOGE("getBufferAndFormat - not executing"); 867 return INVALID_OPERATION; 868 } 869 870 // we do not want mPortBuffers to change during this section 871 // we also don't want mOwnedByClient to change during this 872 Mutex::Autolock al(mBufferLock); 873 874 Vector<BufferInfo> *buffers = &mPortBuffers[portIndex]; 875 if (index >= buffers->size()) { 876 ALOGE("getBufferAndFormat - trying to get buffer with " 877 "bad index (index=%zu buffer_size=%zu)", index, buffers->size()); 878 return INVALID_OPERATION; 879 } 880 881 const BufferInfo &info = buffers->itemAt(index); 882 if (!info.mOwnedByClient) { 883 ALOGE("getBufferAndFormat - invalid operation " 884 "(the index %zu is not owned by client)", index); 885 return INVALID_OPERATION; 886 } 887 888 // by the time buffers array is initialized, crypto is set 889 *buffer = (portIndex == kPortIndexInput && mCrypto != NULL) ? 890 info.mEncryptedData : 891 info.mData; 892 893 *format = info.mFormat; 894 895 return OK; 896} 897 898status_t MediaCodec::flush() { 899 sp<AMessage> msg = new AMessage(kWhatFlush, this); 900 901 sp<AMessage> response; 902 return PostAndAwaitResponse(msg, &response); 903} 904 905status_t MediaCodec::requestIDRFrame() { 906 (new AMessage(kWhatRequestIDRFrame, this))->post(); 907 908 return OK; 909} 910 911void MediaCodec::requestActivityNotification(const sp<AMessage> ¬ify) { 912 sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, this); 913 msg->setMessage("notify", notify); 914 msg->post(); 915} 916 917//////////////////////////////////////////////////////////////////////////////// 918 919void MediaCodec::cancelPendingDequeueOperations() { 920 if (mFlags & kFlagDequeueInputPending) { 921 PostReplyWithError(mDequeueInputReplyID, INVALID_OPERATION); 922 923 ++mDequeueInputTimeoutGeneration; 924 mDequeueInputReplyID = 0; 925 mFlags &= ~kFlagDequeueInputPending; 926 } 927 928 if (mFlags & kFlagDequeueOutputPending) { 929 PostReplyWithError(mDequeueOutputReplyID, INVALID_OPERATION); 930 931 ++mDequeueOutputTimeoutGeneration; 932 mDequeueOutputReplyID = 0; 933 mFlags &= ~kFlagDequeueOutputPending; 934 } 935} 936 937bool MediaCodec::handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest) { 938 if (!isExecuting() || (mFlags & kFlagIsAsync) 939 || (newRequest && (mFlags & kFlagDequeueInputPending))) { 940 PostReplyWithError(replyID, INVALID_OPERATION); 941 return true; 942 } else if (mFlags & kFlagStickyError) { 943 PostReplyWithError(replyID, getStickyError()); 944 return true; 945 } 946 947 ssize_t index = dequeuePortBuffer(kPortIndexInput); 948 949 if (index < 0) { 950 CHECK_EQ(index, -EAGAIN); 951 return false; 952 } 953 954 sp<AMessage> response = new AMessage; 955 response->setSize("index", index); 956 response->postReply(replyID); 957 958 return true; 959} 960 961bool MediaCodec::handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest) { 962 if (!isExecuting() || (mFlags & kFlagIsAsync) 963 || (newRequest && (mFlags & kFlagDequeueOutputPending))) { 964 PostReplyWithError(replyID, INVALID_OPERATION); 965 } else if (mFlags & kFlagStickyError) { 966 PostReplyWithError(replyID, getStickyError()); 967 } else if (mFlags & kFlagOutputBuffersChanged) { 968 PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED); 969 mFlags &= ~kFlagOutputBuffersChanged; 970 } else if (mFlags & kFlagOutputFormatChanged) { 971 PostReplyWithError(replyID, INFO_FORMAT_CHANGED); 972 mFlags &= ~kFlagOutputFormatChanged; 973 } else { 974 sp<AMessage> response = new AMessage; 975 ssize_t index = dequeuePortBuffer(kPortIndexOutput); 976 977 if (index < 0) { 978 CHECK_EQ(index, -EAGAIN); 979 return false; 980 } 981 982 const sp<ABuffer> &buffer = 983 mPortBuffers[kPortIndexOutput].itemAt(index).mData; 984 985 response->setSize("index", index); 986 response->setSize("offset", buffer->offset()); 987 response->setSize("size", buffer->size()); 988 989 int64_t timeUs; 990 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 991 992 response->setInt64("timeUs", timeUs); 993 994 int32_t omxFlags; 995 CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags)); 996 997 uint32_t flags = 0; 998 if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) { 999 flags |= BUFFER_FLAG_SYNCFRAME; 1000 } 1001 if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) { 1002 flags |= BUFFER_FLAG_CODECCONFIG; 1003 } 1004 if (omxFlags & OMX_BUFFERFLAG_EOS) { 1005 flags |= BUFFER_FLAG_EOS; 1006 } 1007 1008 response->setInt32("flags", flags); 1009 response->postReply(replyID); 1010 } 1011 1012 return true; 1013} 1014 1015void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { 1016 switch (msg->what()) { 1017 case kWhatCodecNotify: 1018 { 1019 int32_t what; 1020 CHECK(msg->findInt32("what", &what)); 1021 1022 switch (what) { 1023 case CodecBase::kWhatError: 1024 { 1025 int32_t err, actionCode; 1026 CHECK(msg->findInt32("err", &err)); 1027 CHECK(msg->findInt32("actionCode", &actionCode)); 1028 1029 ALOGE("Codec reported err %#x, actionCode %d, while in state %d", 1030 err, actionCode, mState); 1031 if (err == DEAD_OBJECT) { 1032 mFlags |= kFlagSawMediaServerDie; 1033 mFlags &= ~kFlagIsComponentAllocated; 1034 } 1035 1036 bool sendErrorResponse = true; 1037 1038 switch (mState) { 1039 case INITIALIZING: 1040 { 1041 setState(UNINITIALIZED); 1042 break; 1043 } 1044 1045 case CONFIGURING: 1046 { 1047 setState(actionCode == ACTION_CODE_FATAL ? 1048 UNINITIALIZED : INITIALIZED); 1049 break; 1050 } 1051 1052 case STARTING: 1053 { 1054 setState(actionCode == ACTION_CODE_FATAL ? 1055 UNINITIALIZED : CONFIGURED); 1056 break; 1057 } 1058 1059 case STOPPING: 1060 case RELEASING: 1061 { 1062 // Ignore the error, assuming we'll still get 1063 // the shutdown complete notification. 1064 1065 sendErrorResponse = false; 1066 1067 if (mFlags & kFlagSawMediaServerDie) { 1068 // MediaServer died, there definitely won't 1069 // be a shutdown complete notification after 1070 // all. 1071 1072 // note that we're directly going from 1073 // STOPPING->UNINITIALIZED, instead of the 1074 // usual STOPPING->INITIALIZED state. 1075 setState(UNINITIALIZED); 1076 if (mState == RELEASING) { 1077 mComponentName.clear(); 1078 } 1079 (new AMessage)->postReply(mReplyID); 1080 } 1081 break; 1082 } 1083 1084 case FLUSHING: 1085 { 1086 if (actionCode == ACTION_CODE_FATAL) { 1087 setState(UNINITIALIZED); 1088 } else { 1089 setState( 1090 (mFlags & kFlagIsAsync) ? FLUSHED : STARTED); 1091 } 1092 break; 1093 } 1094 1095 case FLUSHED: 1096 case STARTED: 1097 { 1098 sendErrorResponse = false; 1099 1100 setStickyError(err); 1101 postActivityNotificationIfPossible(); 1102 1103 cancelPendingDequeueOperations(); 1104 1105 if (mFlags & kFlagIsAsync) { 1106 onError(err, actionCode); 1107 } 1108 switch (actionCode) { 1109 case ACTION_CODE_TRANSIENT: 1110 break; 1111 case ACTION_CODE_RECOVERABLE: 1112 setState(INITIALIZED); 1113 break; 1114 default: 1115 setState(UNINITIALIZED); 1116 break; 1117 } 1118 break; 1119 } 1120 1121 default: 1122 { 1123 sendErrorResponse = false; 1124 1125 setStickyError(err); 1126 postActivityNotificationIfPossible(); 1127 1128 // actionCode in an uninitialized state is always fatal. 1129 if (mState == UNINITIALIZED) { 1130 actionCode = ACTION_CODE_FATAL; 1131 } 1132 if (mFlags & kFlagIsAsync) { 1133 onError(err, actionCode); 1134 } 1135 switch (actionCode) { 1136 case ACTION_CODE_TRANSIENT: 1137 break; 1138 case ACTION_CODE_RECOVERABLE: 1139 setState(INITIALIZED); 1140 break; 1141 default: 1142 setState(UNINITIALIZED); 1143 break; 1144 } 1145 break; 1146 } 1147 } 1148 1149 if (sendErrorResponse) { 1150 PostReplyWithError(mReplyID, err); 1151 } 1152 break; 1153 } 1154 1155 case CodecBase::kWhatComponentAllocated: 1156 { 1157 CHECK_EQ(mState, INITIALIZING); 1158 setState(INITIALIZED); 1159 mFlags |= kFlagIsComponentAllocated; 1160 1161 CHECK(msg->findString("componentName", &mComponentName)); 1162 1163 if (mComponentName.startsWith("OMX.google.")) { 1164 mFlags |= kFlagUsesSoftwareRenderer; 1165 } else { 1166 mFlags &= ~kFlagUsesSoftwareRenderer; 1167 } 1168 1169 String8 resourceType; 1170 if (mComponentName.endsWith(".secure")) { 1171 mFlags |= kFlagIsSecure; 1172 resourceType = String8(kResourceSecureCodec); 1173 } else { 1174 mFlags &= ~kFlagIsSecure; 1175 resourceType = String8(kResourceNonSecureCodec); 1176 } 1177 1178 const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec; 1179 addResource(resourceType, String8(subtype), 1); 1180 1181 (new AMessage)->postReply(mReplyID); 1182 break; 1183 } 1184 1185 case CodecBase::kWhatComponentConfigured: 1186 { 1187 if (mState == UNINITIALIZED || mState == INITIALIZED) { 1188 // In case a kWhatError message came in and replied with error, 1189 // we log a warning and ignore. 1190 ALOGW("configure interrupted by error, current state %d", mState); 1191 break; 1192 } 1193 CHECK_EQ(mState, CONFIGURING); 1194 1195 // reset input surface flag 1196 mHaveInputSurface = false; 1197 1198 CHECK(msg->findMessage("input-format", &mInputFormat)); 1199 CHECK(msg->findMessage("output-format", &mOutputFormat)); 1200 1201 int32_t usingSwRenderer; 1202 if (mOutputFormat->findInt32("using-sw-renderer", &usingSwRenderer) 1203 && usingSwRenderer) { 1204 mFlags |= kFlagUsesSoftwareRenderer; 1205 } 1206 setState(CONFIGURED); 1207 (new AMessage)->postReply(mReplyID); 1208 break; 1209 } 1210 1211 case CodecBase::kWhatInputSurfaceCreated: 1212 { 1213 // response to initiateCreateInputSurface() 1214 status_t err = NO_ERROR; 1215 sp<AMessage> response = new AMessage; 1216 if (!msg->findInt32("err", &err)) { 1217 sp<RefBase> obj; 1218 msg->findObject("input-surface", &obj); 1219 CHECK(obj != NULL); 1220 response->setObject("input-surface", obj); 1221 mHaveInputSurface = true; 1222 } else { 1223 response->setInt32("err", err); 1224 } 1225 response->postReply(mReplyID); 1226 break; 1227 } 1228 1229 case CodecBase::kWhatInputSurfaceAccepted: 1230 { 1231 // response to initiateSetInputSurface() 1232 status_t err = NO_ERROR; 1233 sp<AMessage> response = new AMessage(); 1234 if (!msg->findInt32("err", &err)) { 1235 mHaveInputSurface = true; 1236 } else { 1237 response->setInt32("err", err); 1238 } 1239 response->postReply(mReplyID); 1240 break; 1241 } 1242 1243 case CodecBase::kWhatSignaledInputEOS: 1244 { 1245 // response to signalEndOfInputStream() 1246 sp<AMessage> response = new AMessage; 1247 status_t err; 1248 if (msg->findInt32("err", &err)) { 1249 response->setInt32("err", err); 1250 } 1251 response->postReply(mReplyID); 1252 break; 1253 } 1254 1255 1256 case CodecBase::kWhatBuffersAllocated: 1257 { 1258 Mutex::Autolock al(mBufferLock); 1259 int32_t portIndex; 1260 CHECK(msg->findInt32("portIndex", &portIndex)); 1261 1262 ALOGV("%s buffers allocated", 1263 portIndex == kPortIndexInput ? "input" : "output"); 1264 1265 CHECK(portIndex == kPortIndexInput 1266 || portIndex == kPortIndexOutput); 1267 1268 mPortBuffers[portIndex].clear(); 1269 1270 Vector<BufferInfo> *buffers = &mPortBuffers[portIndex]; 1271 1272 sp<RefBase> obj; 1273 CHECK(msg->findObject("portDesc", &obj)); 1274 1275 sp<CodecBase::PortDescription> portDesc = 1276 static_cast<CodecBase::PortDescription *>(obj.get()); 1277 1278 size_t numBuffers = portDesc->countBuffers(); 1279 1280 size_t totalSize = 0; 1281 for (size_t i = 0; i < numBuffers; ++i) { 1282 if (portIndex == kPortIndexInput && mCrypto != NULL) { 1283 totalSize += portDesc->bufferAt(i)->capacity(); 1284 } 1285 } 1286 1287 if (totalSize) { 1288 mDealer = new MemoryDealer(totalSize, "MediaCodec"); 1289 } 1290 1291 for (size_t i = 0; i < numBuffers; ++i) { 1292 BufferInfo info; 1293 info.mBufferID = portDesc->bufferIDAt(i); 1294 info.mOwnedByClient = false; 1295 info.mData = portDesc->bufferAt(i); 1296 1297 if (portIndex == kPortIndexInput && mCrypto != NULL) { 1298 sp<IMemory> mem = mDealer->allocate(info.mData->capacity()); 1299 info.mEncryptedData = 1300 new ABuffer(mem->pointer(), info.mData->capacity()); 1301 info.mSharedEncryptedBuffer = mem; 1302 } 1303 1304 buffers->push_back(info); 1305 } 1306 1307 if (portIndex == kPortIndexOutput) { 1308 if (mState == STARTING) { 1309 // We're always allocating output buffers after 1310 // allocating input buffers, so this is a good 1311 // indication that now all buffers are allocated. 1312 if (mIsVideo) { 1313 String8 subtype; 1314 addResource( 1315 String8(kResourceGraphicMemory), 1316 subtype, 1317 getGraphicBufferSize()); 1318 } 1319 setState(STARTED); 1320 (new AMessage)->postReply(mReplyID); 1321 } else { 1322 mFlags |= kFlagOutputBuffersChanged; 1323 postActivityNotificationIfPossible(); 1324 } 1325 } 1326 break; 1327 } 1328 1329 case CodecBase::kWhatOutputFormatChanged: 1330 { 1331 ALOGV("codec output format changed"); 1332 1333 if (mSoftRenderer == NULL && 1334 mSurface != NULL && 1335 (mFlags & kFlagUsesSoftwareRenderer)) { 1336 AString mime; 1337 CHECK(msg->findString("mime", &mime)); 1338 1339 if (mime.startsWithIgnoreCase("video/")) { 1340 mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees); 1341 } 1342 } 1343 1344 mOutputFormat = msg; 1345 1346 if (mFlags & kFlagIsEncoder) { 1347 // Before we announce the format change we should 1348 // collect codec specific data and amend the output 1349 // format as necessary. 1350 mFlags |= kFlagGatherCodecSpecificData; 1351 } else if (mFlags & kFlagIsAsync) { 1352 onOutputFormatChanged(); 1353 } else { 1354 mFlags |= kFlagOutputFormatChanged; 1355 postActivityNotificationIfPossible(); 1356 } 1357 1358 // Notify mCrypto of video resolution changes 1359 if (mCrypto != NULL) { 1360 int32_t left, top, right, bottom, width, height; 1361 if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) { 1362 mCrypto->notifyResolution(right - left + 1, bottom - top + 1); 1363 } else if (mOutputFormat->findInt32("width", &width) 1364 && mOutputFormat->findInt32("height", &height)) { 1365 mCrypto->notifyResolution(width, height); 1366 } 1367 } 1368 1369 break; 1370 } 1371 1372 case CodecBase::kWhatOutputFramesRendered: 1373 { 1374 // ignore these in all states except running, and check that we have a 1375 // notification set 1376 if (mState == STARTED && mOnFrameRenderedNotification != NULL) { 1377 sp<AMessage> notify = mOnFrameRenderedNotification->dup(); 1378 notify->setMessage("data", msg); 1379 notify->post(); 1380 } 1381 break; 1382 } 1383 1384 case CodecBase::kWhatFillThisBuffer: 1385 { 1386 /* size_t index = */updateBuffers(kPortIndexInput, msg); 1387 1388 if (mState == FLUSHING 1389 || mState == STOPPING 1390 || mState == RELEASING) { 1391 returnBuffersToCodecOnPort(kPortIndexInput); 1392 break; 1393 } 1394 1395 if (!mCSD.empty()) { 1396 ssize_t index = dequeuePortBuffer(kPortIndexInput); 1397 CHECK_GE(index, 0); 1398 1399 // If codec specific data had been specified as 1400 // part of the format in the call to configure and 1401 // if there's more csd left, we submit it here 1402 // clients only get access to input buffers once 1403 // this data has been exhausted. 1404 1405 status_t err = queueCSDInputBuffer(index); 1406 1407 if (err != OK) { 1408 ALOGE("queueCSDInputBuffer failed w/ error %d", 1409 err); 1410 1411 setStickyError(err); 1412 postActivityNotificationIfPossible(); 1413 1414 cancelPendingDequeueOperations(); 1415 } 1416 break; 1417 } 1418 1419 if (mFlags & kFlagIsAsync) { 1420 if (!mHaveInputSurface) { 1421 if (mState == FLUSHED) { 1422 mHavePendingInputBuffers = true; 1423 } else { 1424 onInputBufferAvailable(); 1425 } 1426 } 1427 } else if (mFlags & kFlagDequeueInputPending) { 1428 CHECK(handleDequeueInputBuffer(mDequeueInputReplyID)); 1429 1430 ++mDequeueInputTimeoutGeneration; 1431 mFlags &= ~kFlagDequeueInputPending; 1432 mDequeueInputReplyID = 0; 1433 } else { 1434 postActivityNotificationIfPossible(); 1435 } 1436 break; 1437 } 1438 1439 case CodecBase::kWhatDrainThisBuffer: 1440 { 1441 /* size_t index = */updateBuffers(kPortIndexOutput, msg); 1442 1443 if (mState == FLUSHING 1444 || mState == STOPPING 1445 || mState == RELEASING) { 1446 returnBuffersToCodecOnPort(kPortIndexOutput); 1447 break; 1448 } 1449 1450 sp<ABuffer> buffer; 1451 CHECK(msg->findBuffer("buffer", &buffer)); 1452 1453 int32_t omxFlags; 1454 CHECK(msg->findInt32("flags", &omxFlags)); 1455 1456 buffer->meta()->setInt32("omxFlags", omxFlags); 1457 1458 if (mFlags & kFlagGatherCodecSpecificData) { 1459 // This is the very first output buffer after a 1460 // format change was signalled, it'll either contain 1461 // the one piece of codec specific data we can expect 1462 // or there won't be codec specific data. 1463 if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) { 1464 status_t err = 1465 amendOutputFormatWithCodecSpecificData(buffer); 1466 1467 if (err != OK) { 1468 ALOGE("Codec spit out malformed codec " 1469 "specific data!"); 1470 } 1471 } 1472 1473 mFlags &= ~kFlagGatherCodecSpecificData; 1474 if (mFlags & kFlagIsAsync) { 1475 onOutputFormatChanged(); 1476 } else { 1477 mFlags |= kFlagOutputFormatChanged; 1478 } 1479 } 1480 1481 if (mFlags & kFlagIsAsync) { 1482 onOutputBufferAvailable(); 1483 } else if (mFlags & kFlagDequeueOutputPending) { 1484 CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID)); 1485 1486 ++mDequeueOutputTimeoutGeneration; 1487 mFlags &= ~kFlagDequeueOutputPending; 1488 mDequeueOutputReplyID = 0; 1489 } else { 1490 postActivityNotificationIfPossible(); 1491 } 1492 1493 break; 1494 } 1495 1496 case CodecBase::kWhatEOS: 1497 { 1498 // We already notify the client of this by using the 1499 // corresponding flag in "onOutputBufferReady". 1500 break; 1501 } 1502 1503 case CodecBase::kWhatShutdownCompleted: 1504 { 1505 if (mState == STOPPING) { 1506 setState(INITIALIZED); 1507 } else { 1508 CHECK_EQ(mState, RELEASING); 1509 setState(UNINITIALIZED); 1510 mComponentName.clear(); 1511 } 1512 mFlags &= ~kFlagIsComponentAllocated; 1513 1514 mResourceManagerService->removeResource(getId(mResourceManagerClient)); 1515 1516 (new AMessage)->postReply(mReplyID); 1517 break; 1518 } 1519 1520 case CodecBase::kWhatFlushCompleted: 1521 { 1522 if (mState != FLUSHING) { 1523 ALOGW("received FlushCompleted message in state %d", 1524 mState); 1525 break; 1526 } 1527 1528 if (mFlags & kFlagIsAsync) { 1529 setState(FLUSHED); 1530 } else { 1531 setState(STARTED); 1532 mCodec->signalResume(); 1533 } 1534 1535 (new AMessage)->postReply(mReplyID); 1536 break; 1537 } 1538 1539 default: 1540 TRESPASS(); 1541 } 1542 break; 1543 } 1544 1545 case kWhatInit: 1546 { 1547 sp<AReplyToken> replyID; 1548 CHECK(msg->senderAwaitsResponse(&replyID)); 1549 1550 if (mState != UNINITIALIZED) { 1551 PostReplyWithError(replyID, INVALID_OPERATION); 1552 break; 1553 } 1554 1555 mReplyID = replyID; 1556 setState(INITIALIZING); 1557 1558 AString name; 1559 CHECK(msg->findString("name", &name)); 1560 1561 int32_t nameIsType; 1562 int32_t encoder = false; 1563 CHECK(msg->findInt32("nameIsType", &nameIsType)); 1564 if (nameIsType) { 1565 CHECK(msg->findInt32("encoder", &encoder)); 1566 } 1567 1568 sp<AMessage> format = new AMessage; 1569 1570 if (nameIsType) { 1571 format->setString("mime", name.c_str()); 1572 format->setInt32("encoder", encoder); 1573 } else { 1574 format->setString("componentName", name.c_str()); 1575 } 1576 1577 mCodec->initiateAllocateComponent(format); 1578 break; 1579 } 1580 1581 case kWhatSetNotification: 1582 { 1583 sp<AMessage> notify; 1584 if (msg->findMessage("on-frame-rendered", ¬ify)) { 1585 mOnFrameRenderedNotification = notify; 1586 } 1587 break; 1588 } 1589 1590 case kWhatSetCallback: 1591 { 1592 sp<AReplyToken> replyID; 1593 CHECK(msg->senderAwaitsResponse(&replyID)); 1594 1595 if (mState == UNINITIALIZED 1596 || mState == INITIALIZING 1597 || isExecuting()) { 1598 // callback can't be set after codec is executing, 1599 // or before it's initialized (as the callback 1600 // will be cleared when it goes to INITIALIZED) 1601 PostReplyWithError(replyID, INVALID_OPERATION); 1602 break; 1603 } 1604 1605 sp<AMessage> callback; 1606 CHECK(msg->findMessage("callback", &callback)); 1607 1608 mCallback = callback; 1609 1610 if (mCallback != NULL) { 1611 ALOGI("MediaCodec will operate in async mode"); 1612 mFlags |= kFlagIsAsync; 1613 } else { 1614 mFlags &= ~kFlagIsAsync; 1615 } 1616 1617 sp<AMessage> response = new AMessage; 1618 response->postReply(replyID); 1619 break; 1620 } 1621 1622 case kWhatConfigure: 1623 { 1624 sp<AReplyToken> replyID; 1625 CHECK(msg->senderAwaitsResponse(&replyID)); 1626 1627 if (mState != INITIALIZED) { 1628 PostReplyWithError(replyID, INVALID_OPERATION); 1629 break; 1630 } 1631 1632 sp<RefBase> obj; 1633 CHECK(msg->findObject("surface", &obj)); 1634 1635 sp<AMessage> format; 1636 CHECK(msg->findMessage("format", &format)); 1637 1638 int32_t push; 1639 if (msg->findInt32("push-blank-buffers-on-shutdown", &push) && push != 0) { 1640 mFlags |= kFlagPushBlankBuffersOnShutdown; 1641 } 1642 1643 if (obj != NULL) { 1644 format->setObject("native-window", obj); 1645 status_t err = handleSetSurface(static_cast<Surface *>(obj.get())); 1646 if (err != OK) { 1647 PostReplyWithError(replyID, err); 1648 break; 1649 } 1650 } else { 1651 handleSetSurface(NULL); 1652 } 1653 1654 mReplyID = replyID; 1655 setState(CONFIGURING); 1656 1657 void *crypto; 1658 if (!msg->findPointer("crypto", &crypto)) { 1659 crypto = NULL; 1660 } 1661 1662 mCrypto = static_cast<ICrypto *>(crypto); 1663 1664 uint32_t flags; 1665 CHECK(msg->findInt32("flags", (int32_t *)&flags)); 1666 1667 if (flags & CONFIGURE_FLAG_ENCODE) { 1668 format->setInt32("encoder", true); 1669 mFlags |= kFlagIsEncoder; 1670 } 1671 1672 extractCSD(format); 1673 1674 mCodec->initiateConfigureComponent(format); 1675 break; 1676 } 1677 1678 case kWhatSetSurface: 1679 { 1680 sp<AReplyToken> replyID; 1681 CHECK(msg->senderAwaitsResponse(&replyID)); 1682 1683 status_t err = OK; 1684 sp<Surface> surface; 1685 1686 switch (mState) { 1687 case CONFIGURED: 1688 case STARTED: 1689 case FLUSHED: 1690 { 1691 sp<RefBase> obj; 1692 (void)msg->findObject("surface", &obj); 1693 sp<Surface> surface = static_cast<Surface *>(obj.get()); 1694 if (mSurface == NULL) { 1695 // do not support setting surface if it was not set 1696 err = INVALID_OPERATION; 1697 } else if (obj == NULL) { 1698 // do not support unsetting surface 1699 err = BAD_VALUE; 1700 } else { 1701 err = connectToSurface(surface); 1702 if (err == BAD_VALUE) { 1703 // assuming reconnecting to same surface 1704 // TODO: check if it is the same surface 1705 err = OK; 1706 } else { 1707 if (err == OK) { 1708 if (mFlags & kFlagUsesSoftwareRenderer) { 1709 if (mSoftRenderer != NULL 1710 && (mFlags & kFlagPushBlankBuffersOnShutdown)) { 1711 pushBlankBuffersToNativeWindow(mSurface.get()); 1712 } 1713 mSoftRenderer = new SoftwareRenderer(surface); 1714 // TODO: check if this was successful 1715 } else { 1716 err = mCodec->setSurface(surface); 1717 } 1718 } 1719 if (err == OK) { 1720 (void)disconnectFromSurface(); 1721 mSurface = surface; 1722 } 1723 } 1724 } 1725 break; 1726 } 1727 1728 default: 1729 err = INVALID_OPERATION; 1730 break; 1731 } 1732 1733 PostReplyWithError(replyID, err); 1734 break; 1735 } 1736 1737 case kWhatCreateInputSurface: 1738 case kWhatSetInputSurface: 1739 { 1740 sp<AReplyToken> replyID; 1741 CHECK(msg->senderAwaitsResponse(&replyID)); 1742 1743 // Must be configured, but can't have been started yet. 1744 if (mState != CONFIGURED) { 1745 PostReplyWithError(replyID, INVALID_OPERATION); 1746 break; 1747 } 1748 1749 mReplyID = replyID; 1750 if (msg->what() == kWhatCreateInputSurface) { 1751 mCodec->initiateCreateInputSurface(); 1752 } else { 1753 sp<RefBase> obj; 1754 CHECK(msg->findObject("input-surface", &obj)); 1755 1756 mCodec->initiateSetInputSurface( 1757 static_cast<PersistentSurface *>(obj.get())); 1758 } 1759 break; 1760 } 1761 case kWhatStart: 1762 { 1763 sp<AReplyToken> replyID; 1764 CHECK(msg->senderAwaitsResponse(&replyID)); 1765 1766 if (mState == FLUSHED) { 1767 setState(STARTED); 1768 if (mHavePendingInputBuffers) { 1769 onInputBufferAvailable(); 1770 mHavePendingInputBuffers = false; 1771 } 1772 mCodec->signalResume(); 1773 PostReplyWithError(replyID, OK); 1774 break; 1775 } else if (mState != CONFIGURED) { 1776 PostReplyWithError(replyID, INVALID_OPERATION); 1777 break; 1778 } 1779 1780 mReplyID = replyID; 1781 setState(STARTING); 1782 1783 mCodec->initiateStart(); 1784 break; 1785 } 1786 1787 case kWhatStop: 1788 case kWhatRelease: 1789 { 1790 State targetState = 1791 (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED; 1792 1793 sp<AReplyToken> replyID; 1794 CHECK(msg->senderAwaitsResponse(&replyID)); 1795 1796 // already stopped/released 1797 if (mState == UNINITIALIZED && mReleasedByResourceManager) { 1798 sp<AMessage> response = new AMessage; 1799 response->setInt32("err", OK); 1800 response->postReply(replyID); 1801 break; 1802 } 1803 1804 int32_t reclaimed = 0; 1805 msg->findInt32("reclaimed", &reclaimed); 1806 if (reclaimed) { 1807 mReleasedByResourceManager = true; 1808 } 1809 1810 if (!((mFlags & kFlagIsComponentAllocated) && targetState == UNINITIALIZED) // See 1 1811 && mState != INITIALIZED 1812 && mState != CONFIGURED && !isExecuting()) { 1813 // 1) Permit release to shut down the component if allocated. 1814 // 1815 // 2) We may be in "UNINITIALIZED" state already and 1816 // also shutdown the encoder/decoder without the 1817 // client being aware of this if media server died while 1818 // we were being stopped. The client would assume that 1819 // after stop() returned, it would be safe to call release() 1820 // and it should be in this case, no harm to allow a release() 1821 // if we're already uninitialized. 1822 sp<AMessage> response = new AMessage; 1823 // TODO: we shouldn't throw an exception for stop/release. Change this to wait until 1824 // the previous stop/release completes and then reply with OK. 1825 status_t err = mState == targetState ? OK : INVALID_OPERATION; 1826 response->setInt32("err", err); 1827 if (err == OK && targetState == UNINITIALIZED) { 1828 mComponentName.clear(); 1829 } 1830 response->postReply(replyID); 1831 break; 1832 } 1833 1834 if (mFlags & kFlagSawMediaServerDie) { 1835 // It's dead, Jim. Don't expect initiateShutdown to yield 1836 // any useful results now... 1837 setState(UNINITIALIZED); 1838 if (targetState == UNINITIALIZED) { 1839 mComponentName.clear(); 1840 } 1841 (new AMessage)->postReply(replyID); 1842 break; 1843 } 1844 1845 mReplyID = replyID; 1846 setState(msg->what() == kWhatStop ? STOPPING : RELEASING); 1847 1848 mCodec->initiateShutdown( 1849 msg->what() == kWhatStop /* keepComponentAllocated */); 1850 1851 returnBuffersToCodec(); 1852 1853 if (mSoftRenderer != NULL && (mFlags & kFlagPushBlankBuffersOnShutdown)) { 1854 pushBlankBuffersToNativeWindow(mSurface.get()); 1855 } 1856 break; 1857 } 1858 1859 case kWhatDequeueInputBuffer: 1860 { 1861 sp<AReplyToken> replyID; 1862 CHECK(msg->senderAwaitsResponse(&replyID)); 1863 1864 if (mFlags & kFlagIsAsync) { 1865 ALOGE("dequeueOutputBuffer can't be used in async mode"); 1866 PostReplyWithError(replyID, INVALID_OPERATION); 1867 break; 1868 } 1869 1870 if (mHaveInputSurface) { 1871 ALOGE("dequeueInputBuffer can't be used with input surface"); 1872 PostReplyWithError(replyID, INVALID_OPERATION); 1873 break; 1874 } 1875 1876 if (handleDequeueInputBuffer(replyID, true /* new request */)) { 1877 break; 1878 } 1879 1880 int64_t timeoutUs; 1881 CHECK(msg->findInt64("timeoutUs", &timeoutUs)); 1882 1883 if (timeoutUs == 0ll) { 1884 PostReplyWithError(replyID, -EAGAIN); 1885 break; 1886 } 1887 1888 mFlags |= kFlagDequeueInputPending; 1889 mDequeueInputReplyID = replyID; 1890 1891 if (timeoutUs > 0ll) { 1892 sp<AMessage> timeoutMsg = 1893 new AMessage(kWhatDequeueInputTimedOut, this); 1894 timeoutMsg->setInt32( 1895 "generation", ++mDequeueInputTimeoutGeneration); 1896 timeoutMsg->post(timeoutUs); 1897 } 1898 break; 1899 } 1900 1901 case kWhatDequeueInputTimedOut: 1902 { 1903 int32_t generation; 1904 CHECK(msg->findInt32("generation", &generation)); 1905 1906 if (generation != mDequeueInputTimeoutGeneration) { 1907 // Obsolete 1908 break; 1909 } 1910 1911 CHECK(mFlags & kFlagDequeueInputPending); 1912 1913 PostReplyWithError(mDequeueInputReplyID, -EAGAIN); 1914 1915 mFlags &= ~kFlagDequeueInputPending; 1916 mDequeueInputReplyID = 0; 1917 break; 1918 } 1919 1920 case kWhatQueueInputBuffer: 1921 { 1922 sp<AReplyToken> replyID; 1923 CHECK(msg->senderAwaitsResponse(&replyID)); 1924 1925 if (!isExecuting()) { 1926 PostReplyWithError(replyID, INVALID_OPERATION); 1927 break; 1928 } else if (mFlags & kFlagStickyError) { 1929 PostReplyWithError(replyID, getStickyError()); 1930 break; 1931 } 1932 1933 status_t err = onQueueInputBuffer(msg); 1934 1935 PostReplyWithError(replyID, err); 1936 break; 1937 } 1938 1939 case kWhatDequeueOutputBuffer: 1940 { 1941 sp<AReplyToken> replyID; 1942 CHECK(msg->senderAwaitsResponse(&replyID)); 1943 1944 if (mFlags & kFlagIsAsync) { 1945 ALOGE("dequeueOutputBuffer can't be used in async mode"); 1946 PostReplyWithError(replyID, INVALID_OPERATION); 1947 break; 1948 } 1949 1950 if (handleDequeueOutputBuffer(replyID, true /* new request */)) { 1951 break; 1952 } 1953 1954 int64_t timeoutUs; 1955 CHECK(msg->findInt64("timeoutUs", &timeoutUs)); 1956 1957 if (timeoutUs == 0ll) { 1958 PostReplyWithError(replyID, -EAGAIN); 1959 break; 1960 } 1961 1962 mFlags |= kFlagDequeueOutputPending; 1963 mDequeueOutputReplyID = replyID; 1964 1965 if (timeoutUs > 0ll) { 1966 sp<AMessage> timeoutMsg = 1967 new AMessage(kWhatDequeueOutputTimedOut, this); 1968 timeoutMsg->setInt32( 1969 "generation", ++mDequeueOutputTimeoutGeneration); 1970 timeoutMsg->post(timeoutUs); 1971 } 1972 break; 1973 } 1974 1975 case kWhatDequeueOutputTimedOut: 1976 { 1977 int32_t generation; 1978 CHECK(msg->findInt32("generation", &generation)); 1979 1980 if (generation != mDequeueOutputTimeoutGeneration) { 1981 // Obsolete 1982 break; 1983 } 1984 1985 CHECK(mFlags & kFlagDequeueOutputPending); 1986 1987 PostReplyWithError(mDequeueOutputReplyID, -EAGAIN); 1988 1989 mFlags &= ~kFlagDequeueOutputPending; 1990 mDequeueOutputReplyID = 0; 1991 break; 1992 } 1993 1994 case kWhatReleaseOutputBuffer: 1995 { 1996 sp<AReplyToken> replyID; 1997 CHECK(msg->senderAwaitsResponse(&replyID)); 1998 1999 if (!isExecuting()) { 2000 PostReplyWithError(replyID, INVALID_OPERATION); 2001 break; 2002 } else if (mFlags & kFlagStickyError) { 2003 PostReplyWithError(replyID, getStickyError()); 2004 break; 2005 } 2006 2007 status_t err = onReleaseOutputBuffer(msg); 2008 2009 PostReplyWithError(replyID, err); 2010 break; 2011 } 2012 2013 case kWhatSignalEndOfInputStream: 2014 { 2015 sp<AReplyToken> replyID; 2016 CHECK(msg->senderAwaitsResponse(&replyID)); 2017 2018 if (!isExecuting()) { 2019 PostReplyWithError(replyID, INVALID_OPERATION); 2020 break; 2021 } else if (mFlags & kFlagStickyError) { 2022 PostReplyWithError(replyID, getStickyError()); 2023 break; 2024 } 2025 2026 mReplyID = replyID; 2027 mCodec->signalEndOfInputStream(); 2028 break; 2029 } 2030 2031 case kWhatGetBuffers: 2032 { 2033 sp<AReplyToken> replyID; 2034 CHECK(msg->senderAwaitsResponse(&replyID)); 2035 // Unfortunately widevine legacy source requires knowing all of the 2036 // codec input buffers, so we have to provide them even in async mode. 2037 int32_t widevine = 0; 2038 msg->findInt32("widevine", &widevine); 2039 2040 if (!isExecuting() || ((mFlags & kFlagIsAsync) && !widevine)) { 2041 PostReplyWithError(replyID, INVALID_OPERATION); 2042 break; 2043 } else if (mFlags & kFlagStickyError) { 2044 PostReplyWithError(replyID, getStickyError()); 2045 break; 2046 } 2047 2048 int32_t portIndex; 2049 CHECK(msg->findInt32("portIndex", &portIndex)); 2050 2051 Vector<sp<ABuffer> > *dstBuffers; 2052 CHECK(msg->findPointer("buffers", (void **)&dstBuffers)); 2053 2054 dstBuffers->clear(); 2055 const Vector<BufferInfo> &srcBuffers = mPortBuffers[portIndex]; 2056 2057 for (size_t i = 0; i < srcBuffers.size(); ++i) { 2058 const BufferInfo &info = srcBuffers.itemAt(i); 2059 2060 dstBuffers->push_back( 2061 (portIndex == kPortIndexInput && mCrypto != NULL) 2062 ? info.mEncryptedData : info.mData); 2063 } 2064 2065 (new AMessage)->postReply(replyID); 2066 break; 2067 } 2068 2069 case kWhatFlush: 2070 { 2071 sp<AReplyToken> replyID; 2072 CHECK(msg->senderAwaitsResponse(&replyID)); 2073 2074 if (!isExecuting()) { 2075 PostReplyWithError(replyID, INVALID_OPERATION); 2076 break; 2077 } else if (mFlags & kFlagStickyError) { 2078 PostReplyWithError(replyID, getStickyError()); 2079 break; 2080 } 2081 2082 mReplyID = replyID; 2083 // TODO: skip flushing if already FLUSHED 2084 setState(FLUSHING); 2085 2086 mCodec->signalFlush(); 2087 returnBuffersToCodec(); 2088 break; 2089 } 2090 2091 case kWhatGetInputFormat: 2092 case kWhatGetOutputFormat: 2093 { 2094 sp<AMessage> format = 2095 (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat); 2096 2097 sp<AReplyToken> replyID; 2098 CHECK(msg->senderAwaitsResponse(&replyID)); 2099 2100 if ((mState != CONFIGURED && mState != STARTING && 2101 mState != STARTED && mState != FLUSHING && 2102 mState != FLUSHED) 2103 || format == NULL) { 2104 PostReplyWithError(replyID, INVALID_OPERATION); 2105 break; 2106 } else if (mFlags & kFlagStickyError) { 2107 PostReplyWithError(replyID, getStickyError()); 2108 break; 2109 } 2110 2111 sp<AMessage> response = new AMessage; 2112 response->setMessage("format", format); 2113 response->postReply(replyID); 2114 break; 2115 } 2116 2117 case kWhatRequestIDRFrame: 2118 { 2119 mCodec->signalRequestIDRFrame(); 2120 break; 2121 } 2122 2123 case kWhatRequestActivityNotification: 2124 { 2125 CHECK(mActivityNotify == NULL); 2126 CHECK(msg->findMessage("notify", &mActivityNotify)); 2127 2128 postActivityNotificationIfPossible(); 2129 break; 2130 } 2131 2132 case kWhatGetName: 2133 { 2134 sp<AReplyToken> replyID; 2135 CHECK(msg->senderAwaitsResponse(&replyID)); 2136 2137 if (mComponentName.empty()) { 2138 PostReplyWithError(replyID, INVALID_OPERATION); 2139 break; 2140 } 2141 2142 sp<AMessage> response = new AMessage; 2143 response->setString("name", mComponentName.c_str()); 2144 response->postReply(replyID); 2145 break; 2146 } 2147 2148 case kWhatSetParameters: 2149 { 2150 sp<AReplyToken> replyID; 2151 CHECK(msg->senderAwaitsResponse(&replyID)); 2152 2153 sp<AMessage> params; 2154 CHECK(msg->findMessage("params", ¶ms)); 2155 2156 status_t err = onSetParameters(params); 2157 2158 PostReplyWithError(replyID, err); 2159 break; 2160 } 2161 2162 default: 2163 TRESPASS(); 2164 } 2165} 2166 2167void MediaCodec::extractCSD(const sp<AMessage> &format) { 2168 mCSD.clear(); 2169 2170 size_t i = 0; 2171 for (;;) { 2172 sp<ABuffer> csd; 2173 if (!format->findBuffer(AStringPrintf("csd-%u", i).c_str(), &csd)) { 2174 break; 2175 } 2176 2177 mCSD.push_back(csd); 2178 ++i; 2179 } 2180 2181 ALOGV("Found %zu pieces of codec specific data.", mCSD.size()); 2182} 2183 2184status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) { 2185 CHECK(!mCSD.empty()); 2186 2187 const BufferInfo *info = 2188 &mPortBuffers[kPortIndexInput].itemAt(bufferIndex); 2189 2190 sp<ABuffer> csd = *mCSD.begin(); 2191 mCSD.erase(mCSD.begin()); 2192 2193 const sp<ABuffer> &codecInputData = 2194 (mCrypto != NULL) ? info->mEncryptedData : info->mData; 2195 2196 if (csd->size() > codecInputData->capacity()) { 2197 return -EINVAL; 2198 } 2199 2200 memcpy(codecInputData->data(), csd->data(), csd->size()); 2201 2202 AString errorDetailMsg; 2203 2204 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); 2205 msg->setSize("index", bufferIndex); 2206 msg->setSize("offset", 0); 2207 msg->setSize("size", csd->size()); 2208 msg->setInt64("timeUs", 0ll); 2209 msg->setInt32("flags", BUFFER_FLAG_CODECCONFIG); 2210 msg->setPointer("errorDetailMsg", &errorDetailMsg); 2211 2212 return onQueueInputBuffer(msg); 2213} 2214 2215void MediaCodec::setState(State newState) { 2216 if (newState == INITIALIZED || newState == UNINITIALIZED) { 2217 delete mSoftRenderer; 2218 mSoftRenderer = NULL; 2219 2220 mCrypto.clear(); 2221 handleSetSurface(NULL); 2222 2223 mInputFormat.clear(); 2224 mOutputFormat.clear(); 2225 mFlags &= ~kFlagOutputFormatChanged; 2226 mFlags &= ~kFlagOutputBuffersChanged; 2227 mFlags &= ~kFlagStickyError; 2228 mFlags &= ~kFlagIsEncoder; 2229 mFlags &= ~kFlagGatherCodecSpecificData; 2230 mFlags &= ~kFlagIsAsync; 2231 mStickyError = OK; 2232 2233 mActivityNotify.clear(); 2234 mCallback.clear(); 2235 } 2236 2237 if (newState == UNINITIALIZED) { 2238 // return any straggling buffers, e.g. if we got here on an error 2239 returnBuffersToCodec(); 2240 2241 // The component is gone, mediaserver's probably back up already 2242 // but should definitely be back up should we try to instantiate 2243 // another component.. and the cycle continues. 2244 mFlags &= ~kFlagSawMediaServerDie; 2245 } 2246 2247 mState = newState; 2248 2249 cancelPendingDequeueOperations(); 2250 2251 updateBatteryStat(); 2252} 2253 2254void MediaCodec::returnBuffersToCodec() { 2255 returnBuffersToCodecOnPort(kPortIndexInput); 2256 returnBuffersToCodecOnPort(kPortIndexOutput); 2257} 2258 2259void MediaCodec::returnBuffersToCodecOnPort(int32_t portIndex) { 2260 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); 2261 Mutex::Autolock al(mBufferLock); 2262 2263 Vector<BufferInfo> *buffers = &mPortBuffers[portIndex]; 2264 2265 for (size_t i = 0; i < buffers->size(); ++i) { 2266 BufferInfo *info = &buffers->editItemAt(i); 2267 2268 if (info->mNotify != NULL) { 2269 sp<AMessage> msg = info->mNotify; 2270 info->mNotify = NULL; 2271 info->mOwnedByClient = false; 2272 2273 if (portIndex == kPortIndexInput) { 2274 /* no error, just returning buffers */ 2275 msg->setInt32("err", OK); 2276 } 2277 msg->post(); 2278 } 2279 } 2280 2281 mAvailPortBuffers[portIndex].clear(); 2282} 2283 2284size_t MediaCodec::updateBuffers( 2285 int32_t portIndex, const sp<AMessage> &msg) { 2286 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); 2287 2288 uint32_t bufferID; 2289 CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID)); 2290 2291 Vector<BufferInfo> *buffers = &mPortBuffers[portIndex]; 2292 2293 for (size_t i = 0; i < buffers->size(); ++i) { 2294 BufferInfo *info = &buffers->editItemAt(i); 2295 2296 if (info->mBufferID == bufferID) { 2297 CHECK(info->mNotify == NULL); 2298 CHECK(msg->findMessage("reply", &info->mNotify)); 2299 2300 info->mFormat = 2301 (portIndex == kPortIndexInput) ? mInputFormat : mOutputFormat; 2302 mAvailPortBuffers[portIndex].push_back(i); 2303 2304 return i; 2305 } 2306 } 2307 2308 TRESPASS(); 2309 2310 return 0; 2311} 2312 2313status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) { 2314 size_t index; 2315 size_t offset; 2316 size_t size; 2317 int64_t timeUs; 2318 uint32_t flags; 2319 CHECK(msg->findSize("index", &index)); 2320 CHECK(msg->findSize("offset", &offset)); 2321 CHECK(msg->findInt64("timeUs", &timeUs)); 2322 CHECK(msg->findInt32("flags", (int32_t *)&flags)); 2323 2324 const CryptoPlugin::SubSample *subSamples; 2325 size_t numSubSamples; 2326 const uint8_t *key; 2327 const uint8_t *iv; 2328 CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted; 2329 2330 // We allow the simpler queueInputBuffer API to be used even in 2331 // secure mode, by fabricating a single unencrypted subSample. 2332 CryptoPlugin::SubSample ss; 2333 2334 if (msg->findSize("size", &size)) { 2335 if (mCrypto != NULL) { 2336 ss.mNumBytesOfClearData = size; 2337 ss.mNumBytesOfEncryptedData = 0; 2338 2339 subSamples = &ss; 2340 numSubSamples = 1; 2341 key = NULL; 2342 iv = NULL; 2343 } 2344 } else { 2345 if (mCrypto == NULL) { 2346 return -EINVAL; 2347 } 2348 2349 CHECK(msg->findPointer("subSamples", (void **)&subSamples)); 2350 CHECK(msg->findSize("numSubSamples", &numSubSamples)); 2351 CHECK(msg->findPointer("key", (void **)&key)); 2352 CHECK(msg->findPointer("iv", (void **)&iv)); 2353 2354 int32_t tmp; 2355 CHECK(msg->findInt32("mode", &tmp)); 2356 2357 mode = (CryptoPlugin::Mode)tmp; 2358 2359 size = 0; 2360 for (size_t i = 0; i < numSubSamples; ++i) { 2361 size += subSamples[i].mNumBytesOfClearData; 2362 size += subSamples[i].mNumBytesOfEncryptedData; 2363 } 2364 } 2365 2366 if (index >= mPortBuffers[kPortIndexInput].size()) { 2367 return -ERANGE; 2368 } 2369 2370 BufferInfo *info = &mPortBuffers[kPortIndexInput].editItemAt(index); 2371 2372 if (info->mNotify == NULL || !info->mOwnedByClient) { 2373 return -EACCES; 2374 } 2375 2376 if (offset + size > info->mData->capacity()) { 2377 return -EINVAL; 2378 } 2379 2380 sp<AMessage> reply = info->mNotify; 2381 info->mData->setRange(offset, size); 2382 info->mData->meta()->setInt64("timeUs", timeUs); 2383 2384 if (flags & BUFFER_FLAG_EOS) { 2385 info->mData->meta()->setInt32("eos", true); 2386 } 2387 2388 if (flags & BUFFER_FLAG_CODECCONFIG) { 2389 info->mData->meta()->setInt32("csd", true); 2390 } 2391 2392 if (mCrypto != NULL) { 2393 if (size > info->mEncryptedData->capacity()) { 2394 return -ERANGE; 2395 } 2396 2397 AString *errorDetailMsg; 2398 CHECK(msg->findPointer("errorDetailMsg", (void **)&errorDetailMsg)); 2399 2400 ssize_t result = mCrypto->decrypt( 2401 (mFlags & kFlagIsSecure) != 0, 2402 key, 2403 iv, 2404 mode, 2405 info->mSharedEncryptedBuffer, 2406 offset, 2407 subSamples, 2408 numSubSamples, 2409 info->mData->base(), 2410 errorDetailMsg); 2411 2412 if (result < 0) { 2413 return result; 2414 } 2415 2416 info->mData->setRange(0, result); 2417 } 2418 2419 // synchronization boundary for getBufferAndFormat 2420 { 2421 Mutex::Autolock al(mBufferLock); 2422 info->mOwnedByClient = false; 2423 } 2424 reply->setBuffer("buffer", info->mData); 2425 reply->post(); 2426 2427 info->mNotify = NULL; 2428 2429 return OK; 2430} 2431 2432//static 2433size_t MediaCodec::CreateFramesRenderedMessage( 2434 std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg) { 2435 size_t index = 0; 2436 2437 for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin(); 2438 it != done.cend(); ++it) { 2439 if (it->getRenderTimeNs() < 0) { 2440 continue; // dropped frame from tracking 2441 } 2442 msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs()); 2443 msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs()); 2444 ++index; 2445 } 2446 return index; 2447} 2448 2449status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) { 2450 size_t index; 2451 CHECK(msg->findSize("index", &index)); 2452 2453 int32_t render; 2454 if (!msg->findInt32("render", &render)) { 2455 render = 0; 2456 } 2457 2458 if (!isExecuting()) { 2459 return -EINVAL; 2460 } 2461 2462 if (index >= mPortBuffers[kPortIndexOutput].size()) { 2463 return -ERANGE; 2464 } 2465 2466 BufferInfo *info = &mPortBuffers[kPortIndexOutput].editItemAt(index); 2467 2468 if (info->mNotify == NULL || !info->mOwnedByClient) { 2469 return -EACCES; 2470 } 2471 2472 // synchronization boundary for getBufferAndFormat 2473 { 2474 Mutex::Autolock al(mBufferLock); 2475 info->mOwnedByClient = false; 2476 } 2477 2478 if (render && info->mData != NULL && info->mData->size() != 0) { 2479 info->mNotify->setInt32("render", true); 2480 2481 int64_t mediaTimeUs = -1; 2482 info->mData->meta()->findInt64("timeUs", &mediaTimeUs); 2483 2484 int64_t renderTimeNs = 0; 2485 if (!msg->findInt64("timestampNs", &renderTimeNs)) { 2486 // use media timestamp if client did not request a specific render timestamp 2487 ALOGV("using buffer PTS of %lld", (long long)mediaTimeUs); 2488 renderTimeNs = mediaTimeUs * 1000; 2489 } 2490 info->mNotify->setInt64("timestampNs", renderTimeNs); 2491 2492 if (mSoftRenderer != NULL) { 2493 std::list<FrameRenderTracker::Info> doneFrames = mSoftRenderer->render( 2494 info->mData->data(), info->mData->size(), 2495 mediaTimeUs, renderTimeNs, NULL, info->mFormat); 2496 2497 // if we are running, notify rendered frames 2498 if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) { 2499 sp<AMessage> notify = mOnFrameRenderedNotification->dup(); 2500 sp<AMessage> data = new AMessage; 2501 if (CreateFramesRenderedMessage(doneFrames, data)) { 2502 notify->setMessage("data", data); 2503 notify->post(); 2504 } 2505 } 2506 } 2507 } 2508 2509 info->mNotify->post(); 2510 info->mNotify = NULL; 2511 2512 return OK; 2513} 2514 2515ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) { 2516 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); 2517 2518 List<size_t> *availBuffers = &mAvailPortBuffers[portIndex]; 2519 2520 if (availBuffers->empty()) { 2521 return -EAGAIN; 2522 } 2523 2524 size_t index = *availBuffers->begin(); 2525 availBuffers->erase(availBuffers->begin()); 2526 2527 BufferInfo *info = &mPortBuffers[portIndex].editItemAt(index); 2528 CHECK(!info->mOwnedByClient); 2529 { 2530 Mutex::Autolock al(mBufferLock); 2531 info->mOwnedByClient = true; 2532 2533 // set image-data 2534 if (info->mFormat != NULL) { 2535 sp<ABuffer> imageData; 2536 if (info->mFormat->findBuffer("image-data", &imageData)) { 2537 info->mData->meta()->setBuffer("image-data", imageData); 2538 } 2539 int32_t left, top, right, bottom; 2540 if (info->mFormat->findRect("crop", &left, &top, &right, &bottom)) { 2541 info->mData->meta()->setRect("crop-rect", left, top, right, bottom); 2542 } 2543 } 2544 } 2545 2546 return index; 2547} 2548 2549status_t MediaCodec::connectToSurface(const sp<Surface> &surface) { 2550 status_t err = OK; 2551 if (surface != NULL) { 2552 err = native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA); 2553 if (err == BAD_VALUE) { 2554 ALOGI("native window already connected. Assuming no change of surface"); 2555 return err; 2556 } else if (err == OK) { 2557 // Require a fresh set of buffers after each connect by using a unique generation 2558 // number. Rely on the fact that max supported process id by Linux is 2^22. 2559 // PID is never 0 so we don't have to worry that we use the default generation of 0. 2560 // TODO: come up with a unique scheme if other producers also set the generation number. 2561 static uint32_t mSurfaceGeneration = 0; 2562 uint32_t generation = (getpid() << 10) | (++mSurfaceGeneration & ((1 << 10) - 1)); 2563 surface->setGenerationNumber(generation); 2564 ALOGI("[%s] setting surface generation to %u", mComponentName.c_str(), generation); 2565 2566 // HACK: clear any free buffers. Remove when connect will automatically do this. 2567 // This is needed as the consumer may be holding onto stale frames that it can reattach 2568 // to this surface after disconnect/connect, and those free frames would inherit the new 2569 // generation number. Disconnecting after setting a unique generation prevents this. 2570 native_window_api_disconnect(surface.get(), NATIVE_WINDOW_API_MEDIA); 2571 err = native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA); 2572 } 2573 2574 if (err != OK) { 2575 ALOGE("native_window_api_connect returned an error: %s (%d)", strerror(-err), err); 2576 } 2577 } 2578 return err; 2579} 2580 2581status_t MediaCodec::disconnectFromSurface() { 2582 status_t err = OK; 2583 if (mSurface != NULL) { 2584 // Resetting generation is not technically needed, but there is no need to keep it either 2585 mSurface->setGenerationNumber(0); 2586 err = native_window_api_disconnect(mSurface.get(), NATIVE_WINDOW_API_MEDIA); 2587 if (err != OK) { 2588 ALOGW("native_window_api_disconnect returned an error: %s (%d)", strerror(-err), err); 2589 } 2590 // assume disconnected even on error 2591 mSurface.clear(); 2592 } 2593 return err; 2594} 2595 2596status_t MediaCodec::handleSetSurface(const sp<Surface> &surface) { 2597 status_t err = OK; 2598 if (mSurface != NULL) { 2599 (void)disconnectFromSurface(); 2600 } 2601 if (surface != NULL) { 2602 err = connectToSurface(surface); 2603 if (err == OK) { 2604 mSurface = surface; 2605 } 2606 } 2607 return err; 2608} 2609 2610void MediaCodec::onInputBufferAvailable() { 2611 int32_t index; 2612 while ((index = dequeuePortBuffer(kPortIndexInput)) >= 0) { 2613 sp<AMessage> msg = mCallback->dup(); 2614 msg->setInt32("callbackID", CB_INPUT_AVAILABLE); 2615 msg->setInt32("index", index); 2616 msg->post(); 2617 } 2618} 2619 2620void MediaCodec::onOutputBufferAvailable() { 2621 int32_t index; 2622 while ((index = dequeuePortBuffer(kPortIndexOutput)) >= 0) { 2623 const sp<ABuffer> &buffer = 2624 mPortBuffers[kPortIndexOutput].itemAt(index).mData; 2625 sp<AMessage> msg = mCallback->dup(); 2626 msg->setInt32("callbackID", CB_OUTPUT_AVAILABLE); 2627 msg->setInt32("index", index); 2628 msg->setSize("offset", buffer->offset()); 2629 msg->setSize("size", buffer->size()); 2630 2631 int64_t timeUs; 2632 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 2633 2634 msg->setInt64("timeUs", timeUs); 2635 2636 int32_t omxFlags; 2637 CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags)); 2638 2639 uint32_t flags = 0; 2640 if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) { 2641 flags |= BUFFER_FLAG_SYNCFRAME; 2642 } 2643 if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) { 2644 flags |= BUFFER_FLAG_CODECCONFIG; 2645 } 2646 if (omxFlags & OMX_BUFFERFLAG_EOS) { 2647 flags |= BUFFER_FLAG_EOS; 2648 } 2649 2650 msg->setInt32("flags", flags); 2651 2652 msg->post(); 2653 } 2654} 2655 2656void MediaCodec::onError(status_t err, int32_t actionCode, const char *detail) { 2657 if (mCallback != NULL) { 2658 sp<AMessage> msg = mCallback->dup(); 2659 msg->setInt32("callbackID", CB_ERROR); 2660 msg->setInt32("err", err); 2661 msg->setInt32("actionCode", actionCode); 2662 2663 if (detail != NULL) { 2664 msg->setString("detail", detail); 2665 } 2666 2667 msg->post(); 2668 } 2669} 2670 2671void MediaCodec::onOutputFormatChanged() { 2672 if (mCallback != NULL) { 2673 sp<AMessage> msg = mCallback->dup(); 2674 msg->setInt32("callbackID", CB_OUTPUT_FORMAT_CHANGED); 2675 msg->setMessage("format", mOutputFormat); 2676 msg->post(); 2677 } 2678} 2679 2680 2681void MediaCodec::postActivityNotificationIfPossible() { 2682 if (mActivityNotify == NULL) { 2683 return; 2684 } 2685 2686 bool isErrorOrOutputChanged = 2687 (mFlags & (kFlagStickyError 2688 | kFlagOutputBuffersChanged 2689 | kFlagOutputFormatChanged)); 2690 2691 if (isErrorOrOutputChanged 2692 || !mAvailPortBuffers[kPortIndexInput].empty() 2693 || !mAvailPortBuffers[kPortIndexOutput].empty()) { 2694 mActivityNotify->setInt32("input-buffers", 2695 mAvailPortBuffers[kPortIndexInput].size()); 2696 2697 if (isErrorOrOutputChanged) { 2698 // we want consumer to dequeue as many times as it can 2699 mActivityNotify->setInt32("output-buffers", INT32_MAX); 2700 } else { 2701 mActivityNotify->setInt32("output-buffers", 2702 mAvailPortBuffers[kPortIndexOutput].size()); 2703 } 2704 mActivityNotify->post(); 2705 mActivityNotify.clear(); 2706 } 2707} 2708 2709status_t MediaCodec::setParameters(const sp<AMessage> ¶ms) { 2710 sp<AMessage> msg = new AMessage(kWhatSetParameters, this); 2711 msg->setMessage("params", params); 2712 2713 sp<AMessage> response; 2714 return PostAndAwaitResponse(msg, &response); 2715} 2716 2717status_t MediaCodec::onSetParameters(const sp<AMessage> ¶ms) { 2718 mCodec->signalSetParameters(params); 2719 2720 return OK; 2721} 2722 2723status_t MediaCodec::amendOutputFormatWithCodecSpecificData( 2724 const sp<ABuffer> &buffer) { 2725 AString mime; 2726 CHECK(mOutputFormat->findString("mime", &mime)); 2727 2728 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) { 2729 // Codec specific data should be SPS and PPS in a single buffer, 2730 // each prefixed by a startcode (0x00 0x00 0x00 0x01). 2731 // We separate the two and put them into the output format 2732 // under the keys "csd-0" and "csd-1". 2733 2734 unsigned csdIndex = 0; 2735 2736 const uint8_t *data = buffer->data(); 2737 size_t size = buffer->size(); 2738 2739 const uint8_t *nalStart; 2740 size_t nalSize; 2741 while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) { 2742 sp<ABuffer> csd = new ABuffer(nalSize + 4); 2743 memcpy(csd->data(), "\x00\x00\x00\x01", 4); 2744 memcpy(csd->data() + 4, nalStart, nalSize); 2745 2746 mOutputFormat->setBuffer( 2747 AStringPrintf("csd-%u", csdIndex).c_str(), csd); 2748 2749 ++csdIndex; 2750 } 2751 2752 if (csdIndex != 2) { 2753 return ERROR_MALFORMED; 2754 } 2755 } else { 2756 // For everything else we just stash the codec specific data into 2757 // the output format as a single piece of csd under "csd-0". 2758 mOutputFormat->setBuffer("csd-0", buffer); 2759 } 2760 2761 return OK; 2762} 2763 2764void MediaCodec::updateBatteryStat() { 2765 if (mState == CONFIGURED && !mBatteryStatNotified) { 2766 BatteryNotifier& notifier(BatteryNotifier::getInstance()); 2767 2768 if (mIsVideo) { 2769 notifier.noteStartVideo(); 2770 } else { 2771 notifier.noteStartAudio(); 2772 } 2773 2774 mBatteryStatNotified = true; 2775 } else if (mState == UNINITIALIZED && mBatteryStatNotified) { 2776 BatteryNotifier& notifier(BatteryNotifier::getInstance()); 2777 2778 if (mIsVideo) { 2779 notifier.noteStopVideo(); 2780 } else { 2781 notifier.noteStopAudio(); 2782 } 2783 2784 mBatteryStatNotified = false; 2785 } 2786} 2787 2788} // namespace android 2789