GraphicBufferSource.cpp revision 04f101c35eaa90b1f95939afac30674ec1611e6f
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include <inttypes.h> 18 19#define LOG_TAG "GraphicBufferSource" 20//#define LOG_NDEBUG 0 21#include <utils/Log.h> 22 23#include "GraphicBufferSource.h" 24 25#include <OMX_Core.h> 26#include <media/stagefright/foundation/ADebug.h> 27#include <media/stagefright/foundation/AMessage.h> 28 29#include <media/hardware/MetadataBufferType.h> 30#include <ui/GraphicBuffer.h> 31 32#include <inttypes.h> 33 34namespace android { 35 36static const bool EXTRA_CHECK = true; 37 38 39GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, 40 uint32_t bufferWidth, uint32_t bufferHeight, uint32_t bufferCount, 41 bool useGraphicBufferInMeta) : 42 mInitCheck(UNKNOWN_ERROR), 43 mNodeInstance(nodeInstance), 44 mExecuting(false), 45 mSuspended(false), 46 mNumFramesAvailable(0), 47 mEndOfStream(false), 48 mEndOfStreamSent(false), 49 mMaxTimestampGapUs(-1ll), 50 mPrevOriginalTimeUs(-1ll), 51 mPrevModifiedTimeUs(-1ll), 52 mSkipFramesBeforeNs(-1ll), 53 mRepeatAfterUs(-1ll), 54 mRepeatLastFrameGeneration(0), 55 mRepeatLastFrameTimestamp(-1ll), 56 mLatestSubmittedBufferId(-1), 57 mLatestSubmittedBufferFrameNum(0), 58 mLatestSubmittedBufferUseCount(0), 59 mRepeatBufferDeferred(false), 60 mTimePerCaptureUs(-1ll), 61 mTimePerFrameUs(-1ll), 62 mPrevCaptureUs(-1ll), 63 mPrevFrameUs(-1ll), 64 mUseGraphicBufferInMeta(useGraphicBufferInMeta) { 65 66 ALOGV("GraphicBufferSource w=%u h=%u c=%u", 67 bufferWidth, bufferHeight, bufferCount); 68 69 if (bufferWidth == 0 || bufferHeight == 0) { 70 ALOGE("Invalid dimensions %ux%u", bufferWidth, bufferHeight); 71 mInitCheck = BAD_VALUE; 72 return; 73 } 74 75 String8 name("GraphicBufferSource"); 76 77 BufferQueue::createBufferQueue(&mProducer, &mConsumer); 78 mConsumer->setConsumerName(name); 79 mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight); 80 mConsumer->setConsumerUsageBits(GRALLOC_USAGE_HW_VIDEO_ENCODER); 81 82 mInitCheck = mConsumer->setMaxAcquiredBufferCount(bufferCount); 83 if (mInitCheck != NO_ERROR) { 84 ALOGE("Unable to set BQ max acquired buffer count to %u: %d", 85 bufferCount, mInitCheck); 86 return; 87 } 88 89 // Note that we can't create an sp<...>(this) in a ctor that will not keep a 90 // reference once the ctor ends, as that would cause the refcount of 'this' 91 // dropping to 0 at the end of the ctor. Since all we need is a wp<...> 92 // that's what we create. 93 wp<BufferQueue::ConsumerListener> listener = static_cast<BufferQueue::ConsumerListener*>(this); 94 sp<BufferQueue::ProxyConsumerListener> proxy = new BufferQueue::ProxyConsumerListener(listener); 95 96 mInitCheck = mConsumer->consumerConnect(proxy, false); 97 if (mInitCheck != NO_ERROR) { 98 ALOGE("Error connecting to BufferQueue: %s (%d)", 99 strerror(-mInitCheck), mInitCheck); 100 return; 101 } 102 103 CHECK(mInitCheck == NO_ERROR); 104} 105 106GraphicBufferSource::~GraphicBufferSource() { 107 ALOGV("~GraphicBufferSource"); 108 if (mConsumer != NULL) { 109 status_t err = mConsumer->consumerDisconnect(); 110 if (err != NO_ERROR) { 111 ALOGW("consumerDisconnect failed: %d", err); 112 } 113 } 114} 115 116void GraphicBufferSource::omxExecuting() { 117 Mutex::Autolock autoLock(mMutex); 118 ALOGV("--> executing; avail=%zu, codec vec size=%zd", 119 mNumFramesAvailable, mCodecBuffers.size()); 120 CHECK(!mExecuting); 121 mExecuting = true; 122 123 // Start by loading up as many buffers as possible. We want to do this, 124 // rather than just submit the first buffer, to avoid a degenerate case: 125 // if all BQ buffers arrive before we start executing, and we only submit 126 // one here, the other BQ buffers will just sit until we get notified 127 // that the codec buffer has been released. We'd then acquire and 128 // submit a single additional buffer, repeatedly, never using more than 129 // one codec buffer simultaneously. (We could instead try to submit 130 // all BQ buffers whenever any codec buffer is freed, but if we get the 131 // initial conditions right that will never be useful.) 132 while (mNumFramesAvailable) { 133 if (!fillCodecBuffer_l()) { 134 ALOGV("stop load with frames available (codecAvail=%d)", 135 isCodecBufferAvailable_l()); 136 break; 137 } 138 } 139 140 ALOGV("done loading initial frames, avail=%zu", mNumFramesAvailable); 141 142 // If EOS has already been signaled, and there are no more frames to 143 // submit, try to send EOS now as well. 144 if (mEndOfStream && mNumFramesAvailable == 0) { 145 submitEndOfInputStream_l(); 146 } 147 148 if (mRepeatAfterUs > 0ll && mLooper == NULL) { 149 mReflector = new AHandlerReflector<GraphicBufferSource>(this); 150 151 mLooper = new ALooper; 152 mLooper->registerHandler(mReflector); 153 mLooper->start(); 154 155 if (mLatestSubmittedBufferId >= 0) { 156 sp<AMessage> msg = 157 new AMessage(kWhatRepeatLastFrame, mReflector->id()); 158 159 msg->setInt32("generation", ++mRepeatLastFrameGeneration); 160 msg->post(mRepeatAfterUs); 161 } 162 } 163} 164 165void GraphicBufferSource::omxIdle() { 166 ALOGV("omxIdle"); 167 168 Mutex::Autolock autoLock(mMutex); 169 170 if (mExecuting) { 171 // We are only interested in the transition from executing->idle, 172 // not loaded->idle. 173 mExecuting = false; 174 } 175} 176 177void GraphicBufferSource::omxLoaded(){ 178 Mutex::Autolock autoLock(mMutex); 179 if (!mExecuting) { 180 // This can happen if something failed very early. 181 ALOGW("Dropped back down to Loaded without Executing"); 182 } 183 184 if (mLooper != NULL) { 185 mLooper->unregisterHandler(mReflector->id()); 186 mReflector.clear(); 187 188 mLooper->stop(); 189 mLooper.clear(); 190 } 191 192 ALOGV("--> loaded; avail=%zu eos=%d eosSent=%d", 193 mNumFramesAvailable, mEndOfStream, mEndOfStreamSent); 194 195 // Codec is no longer executing. Discard all codec-related state. 196 mCodecBuffers.clear(); 197 // TODO: scan mCodecBuffers to verify that all mGraphicBuffer entries 198 // are null; complain if not 199 200 mExecuting = false; 201} 202 203void GraphicBufferSource::addCodecBuffer(OMX_BUFFERHEADERTYPE* header) { 204 Mutex::Autolock autoLock(mMutex); 205 206 if (mExecuting) { 207 // This should never happen -- buffers can only be allocated when 208 // transitioning from "loaded" to "idle". 209 ALOGE("addCodecBuffer: buffer added while executing"); 210 return; 211 } 212 213 ALOGV("addCodecBuffer h=%p size=%" PRIu32 " p=%p", 214 header, header->nAllocLen, header->pBuffer); 215 CodecBuffer codecBuffer; 216 codecBuffer.mHeader = header; 217 mCodecBuffers.add(codecBuffer); 218} 219 220void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { 221 Mutex::Autolock autoLock(mMutex); 222 223 if (!mExecuting) { 224 return; 225 } 226 227 int cbi = findMatchingCodecBuffer_l(header); 228 if (cbi < 0) { 229 // This should never happen. 230 ALOGE("codecBufferEmptied: buffer not recognized (h=%p)", header); 231 return; 232 } 233 234 ALOGV("codecBufferEmptied h=%p size=%" PRIu32 " filled=%" PRIu32 " p=%p", 235 header, header->nAllocLen, header->nFilledLen, 236 header->pBuffer); 237 CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi)); 238 239 // header->nFilledLen may not be the original value, so we can't compare 240 // that to zero to see of this was the EOS buffer. Instead we just 241 // see if the GraphicBuffer reference was null, which should only ever 242 // happen for EOS. 243 if (codecBuffer.mGraphicBuffer == NULL) { 244 if (!(mEndOfStream && mEndOfStreamSent)) { 245 // This can happen when broken code sends us the same buffer 246 // twice in a row. 247 ALOGE("ERROR: codecBufferEmptied on non-EOS null buffer " 248 "(buffer emptied twice?)"); 249 } 250 // No GraphicBuffer to deal with, no additional input or output is 251 // expected, so just return. 252 return; 253 } 254 255 if (EXTRA_CHECK) { 256 // Pull the graphic buffer handle back out of the buffer, and confirm 257 // that it matches expectations. 258 OMX_U8* data = header->pBuffer; 259 MetadataBufferType type = *(MetadataBufferType *)data; 260 if (type == kMetadataBufferTypeGrallocSource) { 261 buffer_handle_t bufferHandle; 262 memcpy(&bufferHandle, data + 4, sizeof(buffer_handle_t)); 263 if (bufferHandle != codecBuffer.mGraphicBuffer->handle) { 264 // should never happen 265 ALOGE("codecBufferEmptied: buffer's handle is %p, expected %p", 266 bufferHandle, codecBuffer.mGraphicBuffer->handle); 267 CHECK(!"codecBufferEmptied: mismatched buffer"); 268 } 269 } else if (type == kMetadataBufferTypeGraphicBuffer) { 270 GraphicBuffer *buffer; 271 memcpy(&buffer, data + 4, sizeof(buffer)); 272 if (buffer != codecBuffer.mGraphicBuffer.get()) { 273 // should never happen 274 ALOGE("codecBufferEmptied: buffer is %p, expected %p", 275 buffer, codecBuffer.mGraphicBuffer.get()); 276 CHECK(!"codecBufferEmptied: mismatched buffer"); 277 } 278 } 279 } 280 281 // Find matching entry in our cached copy of the BufferQueue slots. 282 // If we find a match, release that slot. If we don't, the BufferQueue 283 // has dropped that GraphicBuffer, and there's nothing for us to release. 284 int id = codecBuffer.mBuf; 285 if (mBufferSlot[id] != NULL && 286 mBufferSlot[id]->handle == codecBuffer.mGraphicBuffer->handle) { 287 ALOGV("cbi %d matches bq slot %d, handle=%p", 288 cbi, id, mBufferSlot[id]->handle); 289 290 if (id == mLatestSubmittedBufferId) { 291 CHECK_GT(mLatestSubmittedBufferUseCount--, 0); 292 } else { 293 mConsumer->releaseBuffer(id, codecBuffer.mFrameNumber, 294 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); 295 } 296 } else { 297 ALOGV("codecBufferEmptied: no match for emptied buffer in cbi %d", 298 cbi); 299 } 300 301 // Mark the codec buffer as available by clearing the GraphicBuffer ref. 302 codecBuffer.mGraphicBuffer = NULL; 303 304 if (mNumFramesAvailable) { 305 // Fill this codec buffer. 306 CHECK(!mEndOfStreamSent); 307 ALOGV("buffer freed, %zu frames avail (eos=%d)", 308 mNumFramesAvailable, mEndOfStream); 309 fillCodecBuffer_l(); 310 } else if (mEndOfStream) { 311 // No frames available, but EOS is pending, so use this buffer to 312 // send that. 313 ALOGV("buffer freed, EOS pending"); 314 submitEndOfInputStream_l(); 315 } else if (mRepeatBufferDeferred) { 316 bool success = repeatLatestSubmittedBuffer_l(); 317 if (success) { 318 ALOGV("deferred repeatLatestSubmittedBuffer_l SUCCESS"); 319 } else { 320 ALOGV("deferred repeatLatestSubmittedBuffer_l FAILURE"); 321 } 322 mRepeatBufferDeferred = false; 323 } 324 325 return; 326} 327 328void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) { 329 Mutex::Autolock autoLock(mMutex); 330 331 if (mMaxTimestampGapUs > 0ll 332 && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) { 333 ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp); 334 if (index >= 0) { 335 ALOGV("OUT timestamp: %lld -> %lld", 336 static_cast<long long>(header->nTimeStamp), 337 static_cast<long long>(mOriginalTimeUs[index])); 338 header->nTimeStamp = mOriginalTimeUs[index]; 339 mOriginalTimeUs.removeItemsAt(index); 340 } else { 341 // giving up the effort as encoder doesn't appear to preserve pts 342 ALOGW("giving up limiting timestamp gap (pts = %lld)", 343 header->nTimeStamp); 344 mMaxTimestampGapUs = -1ll; 345 } 346 if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) { 347 // something terribly wrong must have happened, giving up... 348 ALOGE("mOriginalTimeUs has too many entries (%zu)", 349 mOriginalTimeUs.size()); 350 mMaxTimestampGapUs = -1ll; 351 } 352 } 353} 354 355void GraphicBufferSource::suspend(bool suspend) { 356 Mutex::Autolock autoLock(mMutex); 357 358 if (suspend) { 359 mSuspended = true; 360 361 while (mNumFramesAvailable > 0) { 362 BufferQueue::BufferItem item; 363 status_t err = mConsumer->acquireBuffer(&item, 0); 364 365 if (err == BufferQueue::NO_BUFFER_AVAILABLE) { 366 // shouldn't happen. 367 ALOGW("suspend: frame was not available"); 368 break; 369 } else if (err != OK) { 370 ALOGW("suspend: acquireBuffer returned err=%d", err); 371 break; 372 } 373 374 --mNumFramesAvailable; 375 376 mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber, 377 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence); 378 } 379 return; 380 } 381 382 mSuspended = false; 383 384 if (mExecuting && mNumFramesAvailable == 0 && mRepeatBufferDeferred) { 385 if (repeatLatestSubmittedBuffer_l()) { 386 ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l SUCCESS"); 387 388 mRepeatBufferDeferred = false; 389 } else { 390 ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l FAILURE"); 391 } 392 } 393} 394 395bool GraphicBufferSource::fillCodecBuffer_l() { 396 CHECK(mExecuting && mNumFramesAvailable > 0); 397 398 if (mSuspended) { 399 return false; 400 } 401 402 int cbi = findAvailableCodecBuffer_l(); 403 if (cbi < 0) { 404 // No buffers available, bail. 405 ALOGV("fillCodecBuffer_l: no codec buffers, avail now %zu", 406 mNumFramesAvailable); 407 return false; 408 } 409 410 ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%zu", 411 mNumFramesAvailable); 412 BufferQueue::BufferItem item; 413 status_t err = mConsumer->acquireBuffer(&item, 0); 414 if (err == BufferQueue::NO_BUFFER_AVAILABLE) { 415 // shouldn't happen 416 ALOGW("fillCodecBuffer_l: frame was not available"); 417 return false; 418 } else if (err != OK) { 419 // now what? fake end-of-stream? 420 ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err); 421 return false; 422 } 423 424 mNumFramesAvailable--; 425 426 // Wait for it to become available. 427 err = item.mFence->waitForever("GraphicBufferSource::fillCodecBuffer_l"); 428 if (err != OK) { 429 ALOGW("failed to wait for buffer fence: %d", err); 430 // keep going 431 } 432 433 // If this is the first time we're seeing this buffer, add it to our 434 // slot table. 435 if (item.mGraphicBuffer != NULL) { 436 ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mBuf); 437 mBufferSlot[item.mBuf] = item.mGraphicBuffer; 438 } 439 440 err = UNKNOWN_ERROR; 441 442 // only submit sample if start time is unspecified, or sample 443 // is queued after the specified start time 444 if (mSkipFramesBeforeNs < 0ll || item.mTimestamp >= mSkipFramesBeforeNs) { 445 // if start time is set, offset time stamp by start time 446 if (mSkipFramesBeforeNs > 0) { 447 item.mTimestamp -= mSkipFramesBeforeNs; 448 } 449 err = submitBuffer_l(item, cbi); 450 } 451 452 if (err != OK) { 453 ALOGV("submitBuffer_l failed, releasing bq buf %d", item.mBuf); 454 mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber, 455 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); 456 } else { 457 ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi); 458 setLatestSubmittedBuffer_l(item); 459 } 460 461 return true; 462} 463 464bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() { 465 CHECK(mExecuting && mNumFramesAvailable == 0); 466 467 if (mLatestSubmittedBufferId < 0 || mSuspended) { 468 return false; 469 } 470 if (mBufferSlot[mLatestSubmittedBufferId] == NULL) { 471 // This can happen if the remote side disconnects, causing 472 // onBuffersReleased() to NULL out our copy of the slots. The 473 // buffer is gone, so we have nothing to show. 474 // 475 // To be on the safe side we try to release the buffer. 476 ALOGD("repeatLatestSubmittedBuffer_l: slot was NULL"); 477 mConsumer->releaseBuffer( 478 mLatestSubmittedBufferId, 479 mLatestSubmittedBufferFrameNum, 480 EGL_NO_DISPLAY, 481 EGL_NO_SYNC_KHR, 482 Fence::NO_FENCE); 483 mLatestSubmittedBufferId = -1; 484 mLatestSubmittedBufferFrameNum = 0; 485 return false; 486 } 487 488 int cbi = findAvailableCodecBuffer_l(); 489 if (cbi < 0) { 490 // No buffers available, bail. 491 ALOGV("repeatLatestSubmittedBuffer_l: no codec buffers."); 492 return false; 493 } 494 495 BufferQueue::BufferItem item; 496 item.mBuf = mLatestSubmittedBufferId; 497 item.mFrameNumber = mLatestSubmittedBufferFrameNum; 498 item.mTimestamp = mRepeatLastFrameTimestamp; 499 500 status_t err = submitBuffer_l(item, cbi); 501 502 if (err != OK) { 503 return false; 504 } 505 506 ++mLatestSubmittedBufferUseCount; 507 508 /* repeat last frame up to kRepeatLastFrameCount times. 509 * in case of static scene, a single repeat might not get rid of encoder 510 * ghosting completely, refresh a couple more times to get better quality 511 */ 512 if (--mRepeatLastFrameCount > 0) { 513 mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000; 514 515 if (mReflector != NULL) { 516 sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id()); 517 msg->setInt32("generation", ++mRepeatLastFrameGeneration); 518 msg->post(mRepeatAfterUs); 519 } 520 } 521 522 return true; 523} 524 525void GraphicBufferSource::setLatestSubmittedBuffer_l( 526 const BufferQueue::BufferItem &item) { 527 ALOGV("setLatestSubmittedBuffer_l"); 528 529 if (mLatestSubmittedBufferId >= 0) { 530 if (mLatestSubmittedBufferUseCount == 0) { 531 mConsumer->releaseBuffer( 532 mLatestSubmittedBufferId, 533 mLatestSubmittedBufferFrameNum, 534 EGL_NO_DISPLAY, 535 EGL_NO_SYNC_KHR, 536 Fence::NO_FENCE); 537 } 538 } 539 540 mLatestSubmittedBufferId = item.mBuf; 541 mLatestSubmittedBufferFrameNum = item.mFrameNumber; 542 mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000; 543 544 mLatestSubmittedBufferUseCount = 1; 545 mRepeatBufferDeferred = false; 546 mRepeatLastFrameCount = kRepeatLastFrameCount; 547 548 if (mReflector != NULL) { 549 sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id()); 550 msg->setInt32("generation", ++mRepeatLastFrameGeneration); 551 msg->post(mRepeatAfterUs); 552 } 553} 554 555status_t GraphicBufferSource::signalEndOfInputStream() { 556 Mutex::Autolock autoLock(mMutex); 557 ALOGV("signalEndOfInputStream: exec=%d avail=%zu eos=%d", 558 mExecuting, mNumFramesAvailable, mEndOfStream); 559 560 if (mEndOfStream) { 561 ALOGE("EOS was already signaled"); 562 return INVALID_OPERATION; 563 } 564 565 // Set the end-of-stream flag. If no frames are pending from the 566 // BufferQueue, and a codec buffer is available, and we're executing, 567 // we initiate the EOS from here. Otherwise, we'll let 568 // codecBufferEmptied() (or omxExecuting) do it. 569 // 570 // Note: if there are no pending frames and all codec buffers are 571 // available, we *must* submit the EOS from here or we'll just 572 // stall since no future events are expected. 573 mEndOfStream = true; 574 575 if (mExecuting && mNumFramesAvailable == 0) { 576 submitEndOfInputStream_l(); 577 } 578 579 return OK; 580} 581 582int64_t GraphicBufferSource::getTimestamp(const BufferQueue::BufferItem &item) { 583 int64_t timeUs = item.mTimestamp / 1000; 584 585 if (mTimePerCaptureUs > 0ll) { 586 // Time lapse or slow motion mode 587 if (mPrevCaptureUs < 0ll) { 588 // first capture 589 mPrevCaptureUs = timeUs; 590 mPrevFrameUs = timeUs; 591 } else { 592 // snap to nearest capture point 593 int64_t nFrames = (timeUs + mTimePerCaptureUs / 2 - mPrevCaptureUs) 594 / mTimePerCaptureUs; 595 if (nFrames <= 0) { 596 // skip this frame as it's too close to previous capture 597 ALOGV("skipping frame, timeUs %lld", static_cast<long long>(timeUs)); 598 return -1; 599 } 600 mPrevCaptureUs = mPrevCaptureUs + nFrames * mTimePerCaptureUs; 601 mPrevFrameUs += mTimePerFrameUs * nFrames; 602 } 603 604 ALOGV("timeUs %lld, captureUs %lld, frameUs %lld", 605 static_cast<long long>(timeUs), 606 static_cast<long long>(mPrevCaptureUs), 607 static_cast<long long>(mPrevFrameUs)); 608 609 return mPrevFrameUs; 610 } else if (mMaxTimestampGapUs > 0ll) { 611 /* Cap timestamp gap between adjacent frames to specified max 612 * 613 * In the scenario of cast mirroring, encoding could be suspended for 614 * prolonged periods. Limiting the pts gap to workaround the problem 615 * where encoder's rate control logic produces huge frames after a 616 * long period of suspension. 617 */ 618 619 int64_t originalTimeUs = timeUs; 620 if (mPrevOriginalTimeUs >= 0ll) { 621 if (originalTimeUs < mPrevOriginalTimeUs) { 622 // Drop the frame if it's going backward in time. Bad timestamp 623 // could disrupt encoder's rate control completely. 624 ALOGW("Dropping frame that's going backward in time"); 625 return -1; 626 } 627 int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs; 628 timeUs = (timestampGapUs < mMaxTimestampGapUs ? 629 timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs; 630 } 631 mPrevOriginalTimeUs = originalTimeUs; 632 mPrevModifiedTimeUs = timeUs; 633 mOriginalTimeUs.add(timeUs, originalTimeUs); 634 ALOGV("IN timestamp: %lld -> %lld", 635 static_cast<long long>(originalTimeUs), 636 static_cast<long long>(timeUs)); 637 } 638 639 return timeUs; 640} 641 642status_t GraphicBufferSource::submitBuffer_l( 643 const BufferQueue::BufferItem &item, int cbi) { 644 ALOGV("submitBuffer_l cbi=%d", cbi); 645 646 int64_t timeUs = getTimestamp(item); 647 if (timeUs < 0ll) { 648 return UNKNOWN_ERROR; 649 } 650 651 CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi)); 652 codecBuffer.mGraphicBuffer = mBufferSlot[item.mBuf]; 653 codecBuffer.mBuf = item.mBuf; 654 codecBuffer.mFrameNumber = item.mFrameNumber; 655 656 OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader; 657 CHECK(header->nAllocLen >= 4 + sizeof(buffer_handle_t)); 658 OMX_U8* data = header->pBuffer; 659 buffer_handle_t handle; 660 if (!mUseGraphicBufferInMeta) { 661 const OMX_U32 type = kMetadataBufferTypeGrallocSource; 662 handle = codecBuffer.mGraphicBuffer->handle; 663 memcpy(data, &type, 4); 664 memcpy(data + 4, &handle, sizeof(buffer_handle_t)); 665 } else { 666 // codecBuffer holds a reference to the GraphicBuffer, so 667 // it is valid while it is with the OMX component 668 const OMX_U32 type = kMetadataBufferTypeGraphicBuffer; 669 memcpy(data, &type, 4); 670 // passing a non-reference-counted graphicBuffer 671 GraphicBuffer *buffer = codecBuffer.mGraphicBuffer.get(); 672 handle = buffer->handle; 673 memcpy(data + 4, &buffer, sizeof(buffer)); 674 } 675 676 status_t err = mNodeInstance->emptyDirectBuffer(header, 0, 677 4 + sizeof(buffer_handle_t), OMX_BUFFERFLAG_ENDOFFRAME, 678 timeUs); 679 if (err != OK) { 680 ALOGW("WARNING: emptyDirectBuffer failed: 0x%x", err); 681 codecBuffer.mGraphicBuffer = NULL; 682 return err; 683 } 684 685 ALOGV("emptyDirectBuffer succeeded, h=%p p=%p bufhandle=%p", 686 header, header->pBuffer, handle); 687 return OK; 688} 689 690void GraphicBufferSource::submitEndOfInputStream_l() { 691 CHECK(mEndOfStream); 692 if (mEndOfStreamSent) { 693 ALOGV("EOS already sent"); 694 return; 695 } 696 697 int cbi = findAvailableCodecBuffer_l(); 698 if (cbi < 0) { 699 ALOGV("submitEndOfInputStream_l: no codec buffers available"); 700 return; 701 } 702 703 // We reject any additional incoming graphic buffers, so there's no need 704 // to stick a placeholder into codecBuffer.mGraphicBuffer to mark it as 705 // in-use. 706 CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi)); 707 708 OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader; 709 if (EXTRA_CHECK) { 710 // Guard against implementations that don't check nFilledLen. 711 size_t fillLen = 4 + sizeof(buffer_handle_t); 712 CHECK(header->nAllocLen >= fillLen); 713 OMX_U8* data = header->pBuffer; 714 memset(data, 0xcd, fillLen); 715 } 716 717 uint64_t timestamp = 0; // does this matter? 718 719 status_t err = mNodeInstance->emptyDirectBuffer(header, /*offset*/ 0, 720 /*length*/ 0, OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_EOS, 721 timestamp); 722 if (err != OK) { 723 ALOGW("emptyDirectBuffer EOS failed: 0x%x", err); 724 } else { 725 ALOGV("submitEndOfInputStream_l: buffer submitted, header=%p cbi=%d", 726 header, cbi); 727 mEndOfStreamSent = true; 728 } 729} 730 731int GraphicBufferSource::findAvailableCodecBuffer_l() { 732 CHECK(mCodecBuffers.size() > 0); 733 734 for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) { 735 if (mCodecBuffers[i].mGraphicBuffer == NULL) { 736 return i; 737 } 738 } 739 return -1; 740} 741 742int GraphicBufferSource::findMatchingCodecBuffer_l( 743 const OMX_BUFFERHEADERTYPE* header) { 744 for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) { 745 if (mCodecBuffers[i].mHeader == header) { 746 return i; 747 } 748 } 749 return -1; 750} 751 752// BufferQueue::ConsumerListener callback 753void GraphicBufferSource::onFrameAvailable(const BufferItem& /*item*/) { 754 Mutex::Autolock autoLock(mMutex); 755 756 ALOGV("onFrameAvailable exec=%d avail=%zu", 757 mExecuting, mNumFramesAvailable); 758 759 if (mEndOfStream || mSuspended) { 760 if (mEndOfStream) { 761 // This should only be possible if a new buffer was queued after 762 // EOS was signaled, i.e. the app is misbehaving. 763 764 ALOGW("onFrameAvailable: EOS is set, ignoring frame"); 765 } else { 766 ALOGV("onFrameAvailable: suspended, ignoring frame"); 767 } 768 769 BufferQueue::BufferItem item; 770 status_t err = mConsumer->acquireBuffer(&item, 0); 771 if (err == OK) { 772 // If this is the first time we're seeing this buffer, add it to our 773 // slot table. 774 if (item.mGraphicBuffer != NULL) { 775 ALOGV("onFrameAvailable: setting mBufferSlot %d", item.mBuf); 776 mBufferSlot[item.mBuf] = item.mGraphicBuffer; 777 } 778 mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber, 779 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence); 780 } 781 return; 782 } 783 784 mNumFramesAvailable++; 785 786 mRepeatBufferDeferred = false; 787 ++mRepeatLastFrameGeneration; 788 789 if (mExecuting) { 790 fillCodecBuffer_l(); 791 } 792} 793 794// BufferQueue::ConsumerListener callback 795void GraphicBufferSource::onBuffersReleased() { 796 Mutex::Autolock lock(mMutex); 797 798 uint64_t slotMask; 799 if (mConsumer->getReleasedBuffers(&slotMask) != NO_ERROR) { 800 ALOGW("onBuffersReleased: unable to get released buffer set"); 801 slotMask = 0xffffffffffffffffULL; 802 } 803 804 ALOGV("onBuffersReleased: 0x%016" PRIx64, slotMask); 805 806 for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) { 807 if ((slotMask & 0x01) != 0) { 808 mBufferSlot[i] = NULL; 809 } 810 slotMask >>= 1; 811 } 812} 813 814// BufferQueue::ConsumerListener callback 815void GraphicBufferSource::onSidebandStreamChanged() { 816 ALOG_ASSERT(false, "GraphicBufferSource can't consume sideband streams"); 817} 818 819status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs( 820 int64_t repeatAfterUs) { 821 Mutex::Autolock autoLock(mMutex); 822 823 if (mExecuting || repeatAfterUs <= 0ll) { 824 return INVALID_OPERATION; 825 } 826 827 mRepeatAfterUs = repeatAfterUs; 828 829 return OK; 830} 831 832status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) { 833 Mutex::Autolock autoLock(mMutex); 834 835 if (mExecuting || maxGapUs <= 0ll) { 836 return INVALID_OPERATION; 837 } 838 839 mMaxTimestampGapUs = maxGapUs; 840 841 return OK; 842} 843 844void GraphicBufferSource::setSkipFramesBeforeUs(int64_t skipFramesBeforeUs) { 845 Mutex::Autolock autoLock(mMutex); 846 847 mSkipFramesBeforeNs = 848 (skipFramesBeforeUs > 0) ? (skipFramesBeforeUs * 1000) : -1ll; 849} 850 851status_t GraphicBufferSource::setTimeLapseUs(int64_t* data) { 852 Mutex::Autolock autoLock(mMutex); 853 854 if (mExecuting || data[0] <= 0ll || data[1] <= 0ll) { 855 return INVALID_OPERATION; 856 } 857 858 mTimePerFrameUs = data[0]; 859 mTimePerCaptureUs = data[1]; 860 861 return OK; 862} 863 864void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) { 865 switch (msg->what()) { 866 case kWhatRepeatLastFrame: 867 { 868 Mutex::Autolock autoLock(mMutex); 869 870 int32_t generation; 871 CHECK(msg->findInt32("generation", &generation)); 872 873 if (generation != mRepeatLastFrameGeneration) { 874 // stale 875 break; 876 } 877 878 if (!mExecuting || mNumFramesAvailable > 0) { 879 break; 880 } 881 882 bool success = repeatLatestSubmittedBuffer_l(); 883 884 if (success) { 885 ALOGV("repeatLatestSubmittedBuffer_l SUCCESS"); 886 } else { 887 ALOGV("repeatLatestSubmittedBuffer_l FAILURE"); 888 mRepeatBufferDeferred = true; 889 } 890 break; 891 } 892 893 default: 894 TRESPASS(); 895 } 896} 897 898} // namespace android 899