Converter.cpp revision 20a5a98667df7ed83e5d9f37e88b1bbdb876d718
1/* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "Converter" 19#include <utils/Log.h> 20 21#include "Converter.h" 22 23#include "MediaPuller.h" 24 25#include <cutils/properties.h> 26#include <gui/SurfaceTextureClient.h> 27#include <media/ICrypto.h> 28#include <media/stagefright/foundation/ABuffer.h> 29#include <media/stagefright/foundation/ADebug.h> 30#include <media/stagefright/foundation/AMessage.h> 31#include <media/stagefright/MediaBuffer.h> 32#include <media/stagefright/MediaCodec.h> 33#include <media/stagefright/MediaDefs.h> 34#include <media/stagefright/MediaErrors.h> 35 36namespace android { 37 38Converter::Converter( 39 const sp<AMessage> ¬ify, 40 const sp<ALooper> &codecLooper, 41 const sp<AMessage> &format, 42 bool usePCMAudio) 43 : mInitCheck(NO_INIT), 44 mNotify(notify), 45 mCodecLooper(codecLooper), 46 mInputFormat(format), 47 mIsVideo(false), 48 mIsPCMAudio(usePCMAudio), 49 mDoMoreWorkPending(false) 50#if ENABLE_SILENCE_DETECTION 51 ,mFirstSilentFrameUs(-1ll) 52 ,mInSilentMode(false) 53#endif 54 { 55 AString mime; 56 CHECK(mInputFormat->findString("mime", &mime)); 57 58 if (!strncasecmp("video/", mime.c_str(), 6)) { 59 mIsVideo = true; 60 } 61 62 CHECK(!usePCMAudio || !mIsVideo); 63 64 mInitCheck = initEncoder(); 65 66 if (mInitCheck != OK) { 67 if (mEncoder != NULL) { 68 mEncoder->release(); 69 mEncoder.clear(); 70 } 71 } 72} 73 74Converter::~Converter() { 75 CHECK(mEncoder == NULL); 76} 77 78void Converter::shutdownAsync() { 79 ALOGV("shutdown"); 80 (new AMessage(kWhatShutdown, id()))->post(); 81} 82 83status_t Converter::initCheck() const { 84 return mInitCheck; 85} 86 87size_t Converter::getInputBufferCount() const { 88 return mEncoderInputBuffers.size(); 89} 90 91sp<AMessage> Converter::getOutputFormat() const { 92 return mOutputFormat; 93} 94 95static int32_t getBitrate(const char *propName, int32_t defaultValue) { 96 char val[PROPERTY_VALUE_MAX]; 97 if (property_get(propName, val, NULL)) { 98 char *end; 99 unsigned long x = strtoul(val, &end, 10); 100 101 if (*end == '\0' && end > val && x > 0) { 102 return x; 103 } 104 } 105 106 return defaultValue; 107} 108 109status_t Converter::initEncoder() { 110 AString inputMIME; 111 CHECK(mInputFormat->findString("mime", &inputMIME)); 112 113 AString outputMIME; 114 bool isAudio = false; 115 if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_AUDIO_RAW)) { 116 if (mIsPCMAudio) { 117 outputMIME = MEDIA_MIMETYPE_AUDIO_RAW; 118 } else { 119 outputMIME = MEDIA_MIMETYPE_AUDIO_AAC; 120 } 121 isAudio = true; 122 } else if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) { 123 outputMIME = MEDIA_MIMETYPE_VIDEO_AVC; 124 } else { 125 TRESPASS(); 126 } 127 128 if (!mIsPCMAudio) { 129 mEncoder = MediaCodec::CreateByType( 130 mCodecLooper, outputMIME.c_str(), true /* encoder */); 131 132 if (mEncoder == NULL) { 133 return ERROR_UNSUPPORTED; 134 } 135 } 136 137 mOutputFormat = mInputFormat->dup(); 138 139 if (mIsPCMAudio) { 140 return OK; 141 } 142 143 mOutputFormat->setString("mime", outputMIME.c_str()); 144 145 int32_t audioBitrate = getBitrate("media.wfd.audio-bitrate", 128000); 146 int32_t videoBitrate = getBitrate("media.wfd.video-bitrate", 2500000); 147 148 ALOGI("using audio bitrate of %d bps, video bitrate of %d bps", 149 audioBitrate, videoBitrate); 150 151 if (isAudio) { 152 mOutputFormat->setInt32("bitrate", audioBitrate); 153 } else { 154 mOutputFormat->setInt32("bitrate", videoBitrate); 155 mOutputFormat->setInt32("frame-rate", 30); 156 mOutputFormat->setInt32("i-frame-interval", 1); // Iframes every 1 secs 157 mOutputFormat->setInt32("prepend-sps-pps-to-idr-frames", 1); 158 } 159 160 ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str()); 161 162 status_t err = mEncoder->configure( 163 mOutputFormat, 164 NULL /* nativeWindow */, 165 NULL /* crypto */, 166 MediaCodec::CONFIGURE_FLAG_ENCODE); 167 168 if (err != OK) { 169 return err; 170 } 171 172 err = mEncoder->start(); 173 174 if (err != OK) { 175 return err; 176 } 177 178 err = mEncoder->getInputBuffers(&mEncoderInputBuffers); 179 180 if (err != OK) { 181 return err; 182 } 183 184 return mEncoder->getOutputBuffers(&mEncoderOutputBuffers); 185} 186 187void Converter::notifyError(status_t err) { 188 sp<AMessage> notify = mNotify->dup(); 189 notify->setInt32("what", kWhatError); 190 notify->setInt32("err", err); 191 notify->post(); 192} 193 194// static 195bool Converter::IsSilence(const sp<ABuffer> &accessUnit) { 196 const uint8_t *ptr = accessUnit->data(); 197 const uint8_t *end = ptr + accessUnit->size(); 198 while (ptr < end) { 199 if (*ptr != 0) { 200 return false; 201 } 202 ++ptr; 203 } 204 205 return true; 206} 207 208void Converter::onMessageReceived(const sp<AMessage> &msg) { 209 switch (msg->what()) { 210 case kWhatMediaPullerNotify: 211 { 212 int32_t what; 213 CHECK(msg->findInt32("what", &what)); 214 215 if (!mIsPCMAudio && mEncoder == NULL) { 216 ALOGV("got msg '%s' after encoder shutdown.", 217 msg->debugString().c_str()); 218 219 if (what == MediaPuller::kWhatAccessUnit) { 220 sp<ABuffer> accessUnit; 221 CHECK(msg->findBuffer("accessUnit", &accessUnit)); 222 223 void *mbuf; 224 if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf) 225 && mbuf != NULL) { 226 ALOGV("releasing mbuf %p", mbuf); 227 228 accessUnit->meta()->setPointer("mediaBuffer", NULL); 229 230 static_cast<MediaBuffer *>(mbuf)->release(); 231 mbuf = NULL; 232 } 233 } 234 break; 235 } 236 237 if (what == MediaPuller::kWhatEOS) { 238 mInputBufferQueue.push_back(NULL); 239 240 feedEncoderInputBuffers(); 241 242 scheduleDoMoreWork(); 243 } else { 244 CHECK_EQ(what, MediaPuller::kWhatAccessUnit); 245 246 sp<ABuffer> accessUnit; 247 CHECK(msg->findBuffer("accessUnit", &accessUnit)); 248 249#if 0 250 void *mbuf; 251 if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf) 252 && mbuf != NULL) { 253 ALOGI("queueing mbuf %p", mbuf); 254 } 255#endif 256 257#if ENABLE_SILENCE_DETECTION 258 if (!mIsVideo) { 259 if (IsSilence(accessUnit)) { 260 if (mInSilentMode) { 261 break; 262 } 263 264 int64_t nowUs = ALooper::GetNowUs(); 265 266 if (mFirstSilentFrameUs < 0ll) { 267 mFirstSilentFrameUs = nowUs; 268 } else if (nowUs >= mFirstSilentFrameUs + 10000000ll) { 269 mInSilentMode = true; 270 ALOGI("audio in silent mode now."); 271 break; 272 } 273 } else { 274 if (mInSilentMode) { 275 ALOGI("audio no longer in silent mode."); 276 } 277 mInSilentMode = false; 278 mFirstSilentFrameUs = -1ll; 279 } 280 } 281#endif 282 283 mInputBufferQueue.push_back(accessUnit); 284 285 feedEncoderInputBuffers(); 286 287 scheduleDoMoreWork(); 288 } 289 break; 290 } 291 292 case kWhatEncoderActivity: 293 { 294#if 0 295 int64_t whenUs; 296 if (msg->findInt64("whenUs", &whenUs)) { 297 int64_t nowUs = ALooper::GetNowUs(); 298 ALOGI("[%s] kWhatEncoderActivity after %lld us", 299 mIsVideo ? "video" : "audio", nowUs - whenUs); 300 } 301#endif 302 303 mDoMoreWorkPending = false; 304 305 if (mEncoder == NULL) { 306 break; 307 } 308 309 status_t err = doMoreWork(); 310 311 if (err != OK) { 312 notifyError(err); 313 } else { 314 scheduleDoMoreWork(); 315 } 316 break; 317 } 318 319 case kWhatRequestIDRFrame: 320 { 321 if (mEncoder == NULL) { 322 break; 323 } 324 325 if (mIsVideo) { 326 ALOGI("requesting IDR frame"); 327 mEncoder->requestIDRFrame(); 328 } 329 break; 330 } 331 332 case kWhatShutdown: 333 { 334 ALOGI("shutting down encoder"); 335 336 if (mEncoder != NULL) { 337 mEncoder->release(); 338 mEncoder.clear(); 339 } 340 341 AString mime; 342 CHECK(mInputFormat->findString("mime", &mime)); 343 ALOGI("encoder (%s) shut down.", mime.c_str()); 344 break; 345 } 346 347 default: 348 TRESPASS(); 349 } 350} 351 352void Converter::scheduleDoMoreWork() { 353 if (mIsPCMAudio) { 354 // There's no encoder involved in this case. 355 return; 356 } 357 358 if (mDoMoreWorkPending) { 359 return; 360 } 361 362 mDoMoreWorkPending = true; 363 364#if 1 365 if (mEncoderActivityNotify == NULL) { 366 mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, id()); 367 } 368 mEncoder->requestActivityNotification(mEncoderActivityNotify->dup()); 369#else 370 sp<AMessage> notify = new AMessage(kWhatEncoderActivity, id()); 371 notify->setInt64("whenUs", ALooper::GetNowUs()); 372 mEncoder->requestActivityNotification(notify); 373#endif 374} 375 376status_t Converter::feedRawAudioInputBuffers() { 377 // Split incoming PCM audio into buffers of 6 AUs of 80 audio frames each 378 // and add a 4 byte header according to the wifi display specs. 379 380 while (!mInputBufferQueue.empty()) { 381 sp<ABuffer> buffer = *mInputBufferQueue.begin(); 382 mInputBufferQueue.erase(mInputBufferQueue.begin()); 383 384 int16_t *ptr = (int16_t *)buffer->data(); 385 int16_t *stop = (int16_t *)(buffer->data() + buffer->size()); 386 while (ptr < stop) { 387 *ptr = htons(*ptr); 388 ++ptr; 389 } 390 391 static const size_t kFrameSize = 2 * sizeof(int16_t); // stereo 392 static const size_t kFramesPerAU = 80; 393 static const size_t kNumAUsPerPESPacket = 6; 394 395 if (mPartialAudioAU != NULL) { 396 size_t bytesMissingForFullAU = 397 kNumAUsPerPESPacket * kFramesPerAU * kFrameSize 398 - mPartialAudioAU->size() + 4; 399 400 size_t copy = buffer->size(); 401 if(copy > bytesMissingForFullAU) { 402 copy = bytesMissingForFullAU; 403 } 404 405 memcpy(mPartialAudioAU->data() + mPartialAudioAU->size(), 406 buffer->data(), 407 copy); 408 409 mPartialAudioAU->setRange(0, mPartialAudioAU->size() + copy); 410 411 buffer->setRange(buffer->offset() + copy, buffer->size() - copy); 412 413 int64_t timeUs; 414 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 415 416 int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0); 417 timeUs += copyUs; 418 buffer->meta()->setInt64("timeUs", timeUs); 419 420 if (bytesMissingForFullAU == copy) { 421 sp<AMessage> notify = mNotify->dup(); 422 notify->setInt32("what", kWhatAccessUnit); 423 notify->setBuffer("accessUnit", mPartialAudioAU); 424 notify->post(); 425 426 mPartialAudioAU.clear(); 427 } 428 } 429 430 while (buffer->size() > 0) { 431 sp<ABuffer> partialAudioAU = 432 new ABuffer( 433 4 434 + kNumAUsPerPESPacket * kFrameSize * kFramesPerAU); 435 436 uint8_t *ptr = partialAudioAU->data(); 437 ptr[0] = 0xa0; // 10100000b 438 ptr[1] = kNumAUsPerPESPacket; 439 ptr[2] = 0; // reserved, audio _emphasis_flag = 0 440 441 static const unsigned kQuantizationWordLength = 0; // 16-bit 442 static const unsigned kAudioSamplingFrequency = 2; // 48Khz 443 static const unsigned kNumberOfAudioChannels = 1; // stereo 444 445 ptr[3] = (kQuantizationWordLength << 6) 446 | (kAudioSamplingFrequency << 3) 447 | kNumberOfAudioChannels; 448 449 size_t copy = buffer->size(); 450 if (copy > partialAudioAU->size() - 4) { 451 copy = partialAudioAU->size() - 4; 452 } 453 454 memcpy(&ptr[4], buffer->data(), copy); 455 456 partialAudioAU->setRange(0, 4 + copy); 457 buffer->setRange(buffer->offset() + copy, buffer->size() - copy); 458 459 int64_t timeUs; 460 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 461 462 partialAudioAU->meta()->setInt64("timeUs", timeUs); 463 464 int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0); 465 timeUs += copyUs; 466 buffer->meta()->setInt64("timeUs", timeUs); 467 468 if (copy == partialAudioAU->size() - 4) { 469 sp<AMessage> notify = mNotify->dup(); 470 notify->setInt32("what", kWhatAccessUnit); 471 notify->setBuffer("accessUnit", partialAudioAU); 472 notify->post(); 473 474 partialAudioAU.clear(); 475 continue; 476 } 477 478 mPartialAudioAU = partialAudioAU; 479 } 480 } 481 482 return OK; 483} 484 485status_t Converter::feedEncoderInputBuffers() { 486 if (mIsPCMAudio) { 487 return feedRawAudioInputBuffers(); 488 } 489 490 while (!mInputBufferQueue.empty() 491 && !mAvailEncoderInputIndices.empty()) { 492 sp<ABuffer> buffer = *mInputBufferQueue.begin(); 493 mInputBufferQueue.erase(mInputBufferQueue.begin()); 494 495 size_t bufferIndex = *mAvailEncoderInputIndices.begin(); 496 mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin()); 497 498 int64_t timeUs = 0ll; 499 uint32_t flags = 0; 500 501 if (buffer != NULL) { 502 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 503 504 memcpy(mEncoderInputBuffers.itemAt(bufferIndex)->data(), 505 buffer->data(), 506 buffer->size()); 507 508 void *mediaBuffer; 509 if (buffer->meta()->findPointer("mediaBuffer", &mediaBuffer) 510 && mediaBuffer != NULL) { 511 mEncoderInputBuffers.itemAt(bufferIndex)->meta() 512 ->setPointer("mediaBuffer", mediaBuffer); 513 514 buffer->meta()->setPointer("mediaBuffer", NULL); 515 } 516 } else { 517 flags = MediaCodec::BUFFER_FLAG_EOS; 518 } 519 520 status_t err = mEncoder->queueInputBuffer( 521 bufferIndex, 0, (buffer == NULL) ? 0 : buffer->size(), 522 timeUs, flags); 523 524 if (err != OK) { 525 return err; 526 } 527 } 528 529 return OK; 530} 531 532status_t Converter::doMoreWork() { 533 status_t err; 534 535 for (;;) { 536 size_t bufferIndex; 537 err = mEncoder->dequeueInputBuffer(&bufferIndex); 538 539 if (err != OK) { 540 break; 541 } 542 543 mAvailEncoderInputIndices.push_back(bufferIndex); 544 } 545 546 feedEncoderInputBuffers(); 547 548 for (;;) { 549 size_t bufferIndex; 550 size_t offset; 551 size_t size; 552 int64_t timeUs; 553 uint32_t flags; 554 err = mEncoder->dequeueOutputBuffer( 555 &bufferIndex, &offset, &size, &timeUs, &flags); 556 557 if (err != OK) { 558 if (err == -EAGAIN) { 559 err = OK; 560 } 561 break; 562 } 563 564 if (flags & MediaCodec::BUFFER_FLAG_EOS) { 565 sp<AMessage> notify = mNotify->dup(); 566 notify->setInt32("what", kWhatEOS); 567 notify->post(); 568 } else { 569 sp<ABuffer> buffer = new ABuffer(size); 570 buffer->meta()->setInt64("timeUs", timeUs); 571 572 ALOGV("[%s] time %lld us (%.2f secs)", 573 mIsVideo ? "video" : "audio", timeUs, timeUs / 1E6); 574 575 memcpy(buffer->data(), 576 mEncoderOutputBuffers.itemAt(bufferIndex)->base() + offset, 577 size); 578 579 if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) { 580 mOutputFormat->setBuffer("csd-0", buffer); 581 } else { 582 sp<AMessage> notify = mNotify->dup(); 583 notify->setInt32("what", kWhatAccessUnit); 584 notify->setBuffer("accessUnit", buffer); 585 notify->post(); 586 } 587 } 588 589 mEncoder->releaseOutputBuffer(bufferIndex); 590 591 if (flags & MediaCodec::BUFFER_FLAG_EOS) { 592 break; 593 } 594 } 595 596 return err; 597} 598 599void Converter::requestIDRFrame() { 600 (new AMessage(kWhatRequestIDRFrame, id()))->post(); 601} 602 603} // namespace android 604