1/* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "Converter" 19#include <utils/Log.h> 20 21#include "Converter.h" 22 23#include "MediaPuller.h" 24 25#include <cutils/properties.h> 26#include <gui/SurfaceTextureClient.h> 27#include <media/ICrypto.h> 28#include <media/stagefright/foundation/ABuffer.h> 29#include <media/stagefright/foundation/ADebug.h> 30#include <media/stagefright/foundation/AMessage.h> 31#include <media/stagefright/MediaBuffer.h> 32#include <media/stagefright/MediaCodec.h> 33#include <media/stagefright/MediaDefs.h> 34#include <media/stagefright/MediaErrors.h> 35 36#include <OMX_Video.h> 37 38namespace android { 39 40Converter::Converter( 41 const sp<AMessage> ¬ify, 42 const sp<ALooper> &codecLooper, 43 const sp<AMessage> &format, 44 bool usePCMAudio) 45 : mInitCheck(NO_INIT), 46 mNotify(notify), 47 mCodecLooper(codecLooper), 48 mInputFormat(format), 49 mIsVideo(false), 50 mIsPCMAudio(usePCMAudio), 51 mNeedToManuallyPrependSPSPPS(false), 52 mDoMoreWorkPending(false) 53#if ENABLE_SILENCE_DETECTION 54 ,mFirstSilentFrameUs(-1ll) 55 ,mInSilentMode(false) 56#endif 57 { 58 AString mime; 59 CHECK(mInputFormat->findString("mime", &mime)); 60 61 if (!strncasecmp("video/", mime.c_str(), 6)) { 62 mIsVideo = true; 63 } 64 65 CHECK(!usePCMAudio || !mIsVideo); 66 67 mInitCheck = initEncoder(); 68 69 if (mInitCheck != OK) { 70 if (mEncoder != NULL) { 71 mEncoder->release(); 72 mEncoder.clear(); 73 } 74 } 75} 76 77Converter::~Converter() { 78 CHECK(mEncoder == NULL); 79} 80 81void Converter::shutdownAsync() { 82 ALOGV("shutdown"); 83 (new AMessage(kWhatShutdown, id()))->post(); 84} 85 86status_t Converter::initCheck() const { 87 return mInitCheck; 88} 89 90size_t Converter::getInputBufferCount() const { 91 return mEncoderInputBuffers.size(); 92} 93 94sp<AMessage> Converter::getOutputFormat() const { 95 return mOutputFormat; 96} 97 98bool Converter::needToManuallyPrependSPSPPS() const { 99 return mNeedToManuallyPrependSPSPPS; 100} 101 102static int32_t getBitrate(const char *propName, int32_t defaultValue) { 103 char val[PROPERTY_VALUE_MAX]; 104 if (property_get(propName, val, NULL)) { 105 char *end; 106 unsigned long x = strtoul(val, &end, 10); 107 108 if (*end == '\0' && end > val && x > 0) { 109 return x; 110 } 111 } 112 113 return defaultValue; 114} 115 116status_t Converter::initEncoder() { 117 AString inputMIME; 118 CHECK(mInputFormat->findString("mime", &inputMIME)); 119 120 AString outputMIME; 121 bool isAudio = false; 122 if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_AUDIO_RAW)) { 123 if (mIsPCMAudio) { 124 outputMIME = MEDIA_MIMETYPE_AUDIO_RAW; 125 } else { 126 outputMIME = MEDIA_MIMETYPE_AUDIO_AAC; 127 } 128 isAudio = true; 129 } else if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) { 130 outputMIME = MEDIA_MIMETYPE_VIDEO_AVC; 131 } else { 132 TRESPASS(); 133 } 134 135 if (!mIsPCMAudio) { 136 mEncoder = MediaCodec::CreateByType( 137 mCodecLooper, outputMIME.c_str(), true /* encoder */); 138 139 if (mEncoder == NULL) { 140 return ERROR_UNSUPPORTED; 141 } 142 } 143 144 mOutputFormat = mInputFormat->dup(); 145 146 if (mIsPCMAudio) { 147 return OK; 148 } 149 150 mOutputFormat->setString("mime", outputMIME.c_str()); 151 152 int32_t audioBitrate = getBitrate("media.wfd.audio-bitrate", 128000); 153 int32_t videoBitrate = getBitrate("media.wfd.video-bitrate", 5000000); 154 155 ALOGI("using audio bitrate of %d bps, video bitrate of %d bps", 156 audioBitrate, videoBitrate); 157 158 if (isAudio) { 159 mOutputFormat->setInt32("bitrate", audioBitrate); 160 } else { 161 mOutputFormat->setInt32("bitrate", videoBitrate); 162 mOutputFormat->setInt32("bitrate-mode", OMX_Video_ControlRateConstant); 163 mOutputFormat->setInt32("frame-rate", 30); 164 mOutputFormat->setInt32("i-frame-interval", 15); // Iframes every 15 secs 165 166 // Configure encoder to use intra macroblock refresh mode 167 mOutputFormat->setInt32("intra-refresh-mode", OMX_VIDEO_IntraRefreshCyclic); 168 169 int width, height, mbs; 170 if (!mOutputFormat->findInt32("width", &width) 171 || !mOutputFormat->findInt32("height", &height)) { 172 return ERROR_UNSUPPORTED; 173 } 174 175 // Update macroblocks in a cyclic fashion with 10% of all MBs within 176 // frame gets updated at one time. It takes about 10 frames to 177 // completely update a whole video frame. If the frame rate is 30, 178 // it takes about 333 ms in the best case (if next frame is not an IDR) 179 // to recover from a lost/corrupted packet. 180 mbs = (((width + 15) / 16) * ((height + 15) / 16) * 10) / 100; 181 mOutputFormat->setInt32("intra-refresh-CIR-mbs", mbs); 182 } 183 184 ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str()); 185 186 mNeedToManuallyPrependSPSPPS = false; 187 188 status_t err = NO_INIT; 189 190 if (!isAudio) { 191 sp<AMessage> tmp = mOutputFormat->dup(); 192 tmp->setInt32("prepend-sps-pps-to-idr-frames", 1); 193 194 err = mEncoder->configure( 195 tmp, 196 NULL /* nativeWindow */, 197 NULL /* crypto */, 198 MediaCodec::CONFIGURE_FLAG_ENCODE); 199 200 if (err == OK) { 201 // Encoder supported prepending SPS/PPS, we don't need to emulate 202 // it. 203 mOutputFormat = tmp; 204 } else { 205 mNeedToManuallyPrependSPSPPS = true; 206 207 ALOGI("We going to manually prepend SPS and PPS to IDR frames."); 208 } 209 } 210 211 if (err != OK) { 212 // We'll get here for audio or if we failed to configure the encoder 213 // to automatically prepend SPS/PPS in the case of video. 214 215 err = mEncoder->configure( 216 mOutputFormat, 217 NULL /* nativeWindow */, 218 NULL /* crypto */, 219 MediaCodec::CONFIGURE_FLAG_ENCODE); 220 } 221 222 if (err != OK) { 223 return err; 224 } 225 226 err = mEncoder->start(); 227 228 if (err != OK) { 229 return err; 230 } 231 232 err = mEncoder->getInputBuffers(&mEncoderInputBuffers); 233 234 if (err != OK) { 235 return err; 236 } 237 238 return mEncoder->getOutputBuffers(&mEncoderOutputBuffers); 239} 240 241void Converter::notifyError(status_t err) { 242 sp<AMessage> notify = mNotify->dup(); 243 notify->setInt32("what", kWhatError); 244 notify->setInt32("err", err); 245 notify->post(); 246} 247 248// static 249bool Converter::IsSilence(const sp<ABuffer> &accessUnit) { 250 const uint8_t *ptr = accessUnit->data(); 251 const uint8_t *end = ptr + accessUnit->size(); 252 while (ptr < end) { 253 if (*ptr != 0) { 254 return false; 255 } 256 ++ptr; 257 } 258 259 return true; 260} 261 262void Converter::onMessageReceived(const sp<AMessage> &msg) { 263 switch (msg->what()) { 264 case kWhatMediaPullerNotify: 265 { 266 int32_t what; 267 CHECK(msg->findInt32("what", &what)); 268 269 if (!mIsPCMAudio && mEncoder == NULL) { 270 ALOGV("got msg '%s' after encoder shutdown.", 271 msg->debugString().c_str()); 272 273 if (what == MediaPuller::kWhatAccessUnit) { 274 sp<ABuffer> accessUnit; 275 CHECK(msg->findBuffer("accessUnit", &accessUnit)); 276 277 void *mbuf; 278 if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf) 279 && mbuf != NULL) { 280 ALOGV("releasing mbuf %p", mbuf); 281 282 accessUnit->meta()->setPointer("mediaBuffer", NULL); 283 284 static_cast<MediaBuffer *>(mbuf)->release(); 285 mbuf = NULL; 286 } 287 } 288 break; 289 } 290 291 if (what == MediaPuller::kWhatEOS) { 292 mInputBufferQueue.push_back(NULL); 293 294 feedEncoderInputBuffers(); 295 296 scheduleDoMoreWork(); 297 } else { 298 CHECK_EQ(what, MediaPuller::kWhatAccessUnit); 299 300 sp<ABuffer> accessUnit; 301 CHECK(msg->findBuffer("accessUnit", &accessUnit)); 302 303#if 0 304 void *mbuf; 305 if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf) 306 && mbuf != NULL) { 307 ALOGI("queueing mbuf %p", mbuf); 308 } 309#endif 310 311#if ENABLE_SILENCE_DETECTION 312 if (!mIsVideo) { 313 if (IsSilence(accessUnit)) { 314 if (mInSilentMode) { 315 break; 316 } 317 318 int64_t nowUs = ALooper::GetNowUs(); 319 320 if (mFirstSilentFrameUs < 0ll) { 321 mFirstSilentFrameUs = nowUs; 322 } else if (nowUs >= mFirstSilentFrameUs + 10000000ll) { 323 mInSilentMode = true; 324 ALOGI("audio in silent mode now."); 325 break; 326 } 327 } else { 328 if (mInSilentMode) { 329 ALOGI("audio no longer in silent mode."); 330 } 331 mInSilentMode = false; 332 mFirstSilentFrameUs = -1ll; 333 } 334 } 335#endif 336 337 mInputBufferQueue.push_back(accessUnit); 338 339 feedEncoderInputBuffers(); 340 341 scheduleDoMoreWork(); 342 } 343 break; 344 } 345 346 case kWhatEncoderActivity: 347 { 348#if 0 349 int64_t whenUs; 350 if (msg->findInt64("whenUs", &whenUs)) { 351 int64_t nowUs = ALooper::GetNowUs(); 352 ALOGI("[%s] kWhatEncoderActivity after %lld us", 353 mIsVideo ? "video" : "audio", nowUs - whenUs); 354 } 355#endif 356 357 mDoMoreWorkPending = false; 358 359 if (mEncoder == NULL) { 360 break; 361 } 362 363 status_t err = doMoreWork(); 364 365 if (err != OK) { 366 notifyError(err); 367 } else { 368 scheduleDoMoreWork(); 369 } 370 break; 371 } 372 373 case kWhatRequestIDRFrame: 374 { 375 if (mEncoder == NULL) { 376 break; 377 } 378 379 if (mIsVideo) { 380 ALOGI("requesting IDR frame"); 381 mEncoder->requestIDRFrame(); 382 } 383 break; 384 } 385 386 case kWhatShutdown: 387 { 388 ALOGI("shutting down encoder"); 389 390 if (mEncoder != NULL) { 391 mEncoder->release(); 392 mEncoder.clear(); 393 } 394 395 AString mime; 396 CHECK(mInputFormat->findString("mime", &mime)); 397 ALOGI("encoder (%s) shut down.", mime.c_str()); 398 break; 399 } 400 401 default: 402 TRESPASS(); 403 } 404} 405 406void Converter::scheduleDoMoreWork() { 407 if (mIsPCMAudio) { 408 // There's no encoder involved in this case. 409 return; 410 } 411 412 if (mDoMoreWorkPending) { 413 return; 414 } 415 416 mDoMoreWorkPending = true; 417 418#if 1 419 if (mEncoderActivityNotify == NULL) { 420 mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, id()); 421 } 422 mEncoder->requestActivityNotification(mEncoderActivityNotify->dup()); 423#else 424 sp<AMessage> notify = new AMessage(kWhatEncoderActivity, id()); 425 notify->setInt64("whenUs", ALooper::GetNowUs()); 426 mEncoder->requestActivityNotification(notify); 427#endif 428} 429 430status_t Converter::feedRawAudioInputBuffers() { 431 // Split incoming PCM audio into buffers of 6 AUs of 80 audio frames each 432 // and add a 4 byte header according to the wifi display specs. 433 434 while (!mInputBufferQueue.empty()) { 435 sp<ABuffer> buffer = *mInputBufferQueue.begin(); 436 mInputBufferQueue.erase(mInputBufferQueue.begin()); 437 438 int16_t *ptr = (int16_t *)buffer->data(); 439 int16_t *stop = (int16_t *)(buffer->data() + buffer->size()); 440 while (ptr < stop) { 441 *ptr = htons(*ptr); 442 ++ptr; 443 } 444 445 static const size_t kFrameSize = 2 * sizeof(int16_t); // stereo 446 static const size_t kFramesPerAU = 80; 447 static const size_t kNumAUsPerPESPacket = 6; 448 449 if (mPartialAudioAU != NULL) { 450 size_t bytesMissingForFullAU = 451 kNumAUsPerPESPacket * kFramesPerAU * kFrameSize 452 - mPartialAudioAU->size() + 4; 453 454 size_t copy = buffer->size(); 455 if(copy > bytesMissingForFullAU) { 456 copy = bytesMissingForFullAU; 457 } 458 459 memcpy(mPartialAudioAU->data() + mPartialAudioAU->size(), 460 buffer->data(), 461 copy); 462 463 mPartialAudioAU->setRange(0, mPartialAudioAU->size() + copy); 464 465 buffer->setRange(buffer->offset() + copy, buffer->size() - copy); 466 467 int64_t timeUs; 468 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 469 470 int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0); 471 timeUs += copyUs; 472 buffer->meta()->setInt64("timeUs", timeUs); 473 474 if (bytesMissingForFullAU == copy) { 475 sp<AMessage> notify = mNotify->dup(); 476 notify->setInt32("what", kWhatAccessUnit); 477 notify->setBuffer("accessUnit", mPartialAudioAU); 478 notify->post(); 479 480 mPartialAudioAU.clear(); 481 } 482 } 483 484 while (buffer->size() > 0) { 485 sp<ABuffer> partialAudioAU = 486 new ABuffer( 487 4 488 + kNumAUsPerPESPacket * kFrameSize * kFramesPerAU); 489 490 uint8_t *ptr = partialAudioAU->data(); 491 ptr[0] = 0xa0; // 10100000b 492 ptr[1] = kNumAUsPerPESPacket; 493 ptr[2] = 0; // reserved, audio _emphasis_flag = 0 494 495 static const unsigned kQuantizationWordLength = 0; // 16-bit 496 static const unsigned kAudioSamplingFrequency = 2; // 48Khz 497 static const unsigned kNumberOfAudioChannels = 1; // stereo 498 499 ptr[3] = (kQuantizationWordLength << 6) 500 | (kAudioSamplingFrequency << 3) 501 | kNumberOfAudioChannels; 502 503 size_t copy = buffer->size(); 504 if (copy > partialAudioAU->size() - 4) { 505 copy = partialAudioAU->size() - 4; 506 } 507 508 memcpy(&ptr[4], buffer->data(), copy); 509 510 partialAudioAU->setRange(0, 4 + copy); 511 buffer->setRange(buffer->offset() + copy, buffer->size() - copy); 512 513 int64_t timeUs; 514 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 515 516 partialAudioAU->meta()->setInt64("timeUs", timeUs); 517 518 int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0); 519 timeUs += copyUs; 520 buffer->meta()->setInt64("timeUs", timeUs); 521 522 if (copy == partialAudioAU->capacity() - 4) { 523 sp<AMessage> notify = mNotify->dup(); 524 notify->setInt32("what", kWhatAccessUnit); 525 notify->setBuffer("accessUnit", partialAudioAU); 526 notify->post(); 527 528 partialAudioAU.clear(); 529 continue; 530 } 531 532 mPartialAudioAU = partialAudioAU; 533 } 534 } 535 536 return OK; 537} 538 539status_t Converter::feedEncoderInputBuffers() { 540 if (mIsPCMAudio) { 541 return feedRawAudioInputBuffers(); 542 } 543 544 while (!mInputBufferQueue.empty() 545 && !mAvailEncoderInputIndices.empty()) { 546 sp<ABuffer> buffer = *mInputBufferQueue.begin(); 547 mInputBufferQueue.erase(mInputBufferQueue.begin()); 548 549 size_t bufferIndex = *mAvailEncoderInputIndices.begin(); 550 mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin()); 551 552 int64_t timeUs = 0ll; 553 uint32_t flags = 0; 554 555 if (buffer != NULL) { 556 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 557 558 memcpy(mEncoderInputBuffers.itemAt(bufferIndex)->data(), 559 buffer->data(), 560 buffer->size()); 561 562 void *mediaBuffer; 563 if (buffer->meta()->findPointer("mediaBuffer", &mediaBuffer) 564 && mediaBuffer != NULL) { 565 mEncoderInputBuffers.itemAt(bufferIndex)->meta() 566 ->setPointer("mediaBuffer", mediaBuffer); 567 568 buffer->meta()->setPointer("mediaBuffer", NULL); 569 } 570 } else { 571 flags = MediaCodec::BUFFER_FLAG_EOS; 572 } 573 574 status_t err = mEncoder->queueInputBuffer( 575 bufferIndex, 0, (buffer == NULL) ? 0 : buffer->size(), 576 timeUs, flags); 577 578 if (err != OK) { 579 return err; 580 } 581 } 582 583 return OK; 584} 585 586status_t Converter::doMoreWork() { 587 status_t err; 588 589 for (;;) { 590 size_t bufferIndex; 591 err = mEncoder->dequeueInputBuffer(&bufferIndex); 592 593 if (err != OK) { 594 break; 595 } 596 597 mAvailEncoderInputIndices.push_back(bufferIndex); 598 } 599 600 feedEncoderInputBuffers(); 601 602 for (;;) { 603 size_t bufferIndex; 604 size_t offset; 605 size_t size; 606 int64_t timeUs; 607 uint32_t flags; 608 err = mEncoder->dequeueOutputBuffer( 609 &bufferIndex, &offset, &size, &timeUs, &flags); 610 611 if (err != OK) { 612 if (err == -EAGAIN) { 613 err = OK; 614 } 615 break; 616 } 617 618 if (flags & MediaCodec::BUFFER_FLAG_EOS) { 619 sp<AMessage> notify = mNotify->dup(); 620 notify->setInt32("what", kWhatEOS); 621 notify->post(); 622 } else { 623 sp<ABuffer> buffer = new ABuffer(size); 624 buffer->meta()->setInt64("timeUs", timeUs); 625 626 ALOGV("[%s] time %lld us (%.2f secs)", 627 mIsVideo ? "video" : "audio", timeUs, timeUs / 1E6); 628 629 memcpy(buffer->data(), 630 mEncoderOutputBuffers.itemAt(bufferIndex)->base() + offset, 631 size); 632 633 if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) { 634 mOutputFormat->setBuffer("csd-0", buffer); 635 } else { 636 sp<AMessage> notify = mNotify->dup(); 637 notify->setInt32("what", kWhatAccessUnit); 638 notify->setBuffer("accessUnit", buffer); 639 notify->post(); 640 } 641 } 642 643 mEncoder->releaseOutputBuffer(bufferIndex); 644 645 if (flags & MediaCodec::BUFFER_FLAG_EOS) { 646 break; 647 } 648 } 649 650 return err; 651} 652 653void Converter::requestIDRFrame() { 654 (new AMessage(kWhatRequestIDRFrame, id()))->post(); 655} 656 657} // namespace android 658