Converter.cpp revision 2aea9552aeba92bbaf9e56c666049ea2d14057b5 
1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "Converter"
19#include <utils/Log.h>
20
21#include "Converter.h"
22
23#include "MediaPuller.h"
24
25#include <cutils/properties.h>
26#include <gui/Surface.h>
27#include <media/ICrypto.h>
28#include <media/stagefright/foundation/ABuffer.h>
29#include <media/stagefright/foundation/ADebug.h>
30#include <media/stagefright/foundation/AMessage.h>
31#include <media/stagefright/MediaBuffer.h>
32#include <media/stagefright/MediaCodec.h>
33#include <media/stagefright/MediaDefs.h>
34#include <media/stagefright/MediaErrors.h>
35
36#include <OMX_Video.h>
37
38namespace android {
39
40Converter::Converter(
41        const sp<AMessage> &notify,
42        const sp<ALooper> &codecLooper,
43        const sp<AMessage> &format,
44        bool usePCMAudio)
45    : mInitCheck(NO_INIT),
46      mNotify(notify),
47      mCodecLooper(codecLooper),
48      mInputFormat(format),
49      mIsVideo(false),
50      mIsPCMAudio(usePCMAudio),
51      mNeedToManuallyPrependSPSPPS(false),
52      mDoMoreWorkPending(false)
53#if ENABLE_SILENCE_DETECTION
54      ,mFirstSilentFrameUs(-1ll)
55      ,mInSilentMode(false)
56#endif
57      ,mPrevVideoBitrate(-1)
58      ,mNumFramesToDrop(0)
59    {
60    AString mime;
61    CHECK(mInputFormat->findString("mime", &mime));
62
63    if (!strncasecmp("video/", mime.c_str(), 6)) {
64        mIsVideo = true;
65    }
66
67    CHECK(!usePCMAudio || !mIsVideo);
68
69    mInitCheck = initEncoder();
70
71    if (mInitCheck != OK) {
72        releaseEncoder();
73    }
74}
75
76static void ReleaseMediaBufferReference(const sp<ABuffer> &accessUnit) {
77    void *mbuf;
78    if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf)
79            && mbuf != NULL) {
80        ALOGV("releasing mbuf %p", mbuf);
81
82        accessUnit->meta()->setPointer("mediaBuffer", NULL);
83
84        static_cast<MediaBuffer *>(mbuf)->release();
85        mbuf = NULL;
86    }
87}
88
89void Converter::releaseEncoder() {
90    if (mEncoder == NULL) {
91        return;
92    }
93
94    mEncoder->release();
95    mEncoder.clear();
96
97    while (!mInputBufferQueue.empty()) {
98        sp<ABuffer> accessUnit = *mInputBufferQueue.begin();
99        mInputBufferQueue.erase(mInputBufferQueue.begin());
100
101        ReleaseMediaBufferReference(accessUnit);
102    }
103
104    for (size_t i = 0; i < mEncoderInputBuffers.size(); ++i) {
105        sp<ABuffer> accessUnit = mEncoderInputBuffers.itemAt(i);
106        ReleaseMediaBufferReference(accessUnit);
107    }
108
109    mEncoderInputBuffers.clear();
110    mEncoderOutputBuffers.clear();
111}
112
113Converter::~Converter() {
114    CHECK(mEncoder == NULL);
115}
116
117void Converter::shutdownAsync() {
118    ALOGV("shutdown");
119    (new AMessage(kWhatShutdown, id()))->post();
120}
121
122status_t Converter::initCheck() const {
123    return mInitCheck;
124}
125
126size_t Converter::getInputBufferCount() const {
127    return mEncoderInputBuffers.size();
128}
129
130sp<AMessage> Converter::getOutputFormat() const {
131    return mOutputFormat;
132}
133
134bool Converter::needToManuallyPrependSPSPPS() const {
135    return mNeedToManuallyPrependSPSPPS;
136}
137
138static int32_t getBitrate(const char *propName, int32_t defaultValue) {
139    char val[PROPERTY_VALUE_MAX];
140    if (property_get(propName, val, NULL)) {
141        char *end;
142        unsigned long x = strtoul(val, &end, 10);
143
144        if (*end == '\0' && end > val && x > 0) {
145            return x;
146        }
147    }
148
149    return defaultValue;
150}
151
152status_t Converter::initEncoder() {
153    AString inputMIME;
154    CHECK(mInputFormat->findString("mime", &inputMIME));
155
156    AString outputMIME;
157    bool isAudio = false;
158    if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_AUDIO_RAW)) {
159        if (mIsPCMAudio) {
160            outputMIME = MEDIA_MIMETYPE_AUDIO_RAW;
161        } else {
162            outputMIME = MEDIA_MIMETYPE_AUDIO_AAC;
163        }
164        isAudio = true;
165    } else if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) {
166        outputMIME = MEDIA_MIMETYPE_VIDEO_AVC;
167    } else {
168        TRESPASS();
169    }
170
171    if (!mIsPCMAudio) {
172        mEncoder = MediaCodec::CreateByType(
173                mCodecLooper, outputMIME.c_str(), true /* encoder */);
174
175        if (mEncoder == NULL) {
176            return ERROR_UNSUPPORTED;
177        }
178    }
179
180    mOutputFormat = mInputFormat->dup();
181
182    if (mIsPCMAudio) {
183        return OK;
184    }
185
186    mOutputFormat->setString("mime", outputMIME.c_str());
187
188    int32_t audioBitrate = getBitrate("media.wfd.audio-bitrate", 128000);
189    int32_t videoBitrate = getBitrate("media.wfd.video-bitrate", 5000000);
190    mPrevVideoBitrate = videoBitrate;
191
192    ALOGI("using audio bitrate of %d bps, video bitrate of %d bps",
193          audioBitrate, videoBitrate);
194
195    if (isAudio) {
196        mOutputFormat->setInt32("bitrate", audioBitrate);
197    } else {
198        mOutputFormat->setInt32("bitrate", videoBitrate);
199        mOutputFormat->setInt32("bitrate-mode", OMX_Video_ControlRateConstant);
200        mOutputFormat->setInt32("frame-rate", 30);
201        mOutputFormat->setInt32("i-frame-interval", 15);  // Iframes every 15 secs
202
203        // Configure encoder to use intra macroblock refresh mode
204        mOutputFormat->setInt32("intra-refresh-mode", OMX_VIDEO_IntraRefreshCyclic);
205
206        int width, height, mbs;
207        if (!mOutputFormat->findInt32("width", &width)
208                || !mOutputFormat->findInt32("height", &height)) {
209            return ERROR_UNSUPPORTED;
210        }
211
212        // Update macroblocks in a cyclic fashion with 10% of all MBs within
213        // frame gets updated at one time. It takes about 10 frames to
214        // completely update a whole video frame. If the frame rate is 30,
215        // it takes about 333 ms in the best case (if next frame is not an IDR)
216        // to recover from a lost/corrupted packet.
217        mbs = (((width + 15) / 16) * ((height + 15) / 16) * 10) / 100;
218        mOutputFormat->setInt32("intra-refresh-CIR-mbs", mbs);
219    }
220
221    ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str());
222
223    mNeedToManuallyPrependSPSPPS = false;
224
225    status_t err = NO_INIT;
226
227    if (!isAudio) {
228        sp<AMessage> tmp = mOutputFormat->dup();
229        tmp->setInt32("prepend-sps-pps-to-idr-frames", 1);
230
231        err = mEncoder->configure(
232                tmp,
233                NULL /* nativeWindow */,
234                NULL /* crypto */,
235                MediaCodec::CONFIGURE_FLAG_ENCODE);
236
237        if (err == OK) {
238            // Encoder supported prepending SPS/PPS, we don't need to emulate
239            // it.
240            mOutputFormat = tmp;
241        } else {
242            mNeedToManuallyPrependSPSPPS = true;
243
244            ALOGI("We going to manually prepend SPS and PPS to IDR frames.");
245        }
246    }
247
248    if (err != OK) {
249        // We'll get here for audio or if we failed to configure the encoder
250        // to automatically prepend SPS/PPS in the case of video.
251
252        err = mEncoder->configure(
253                    mOutputFormat,
254                    NULL /* nativeWindow */,
255                    NULL /* crypto */,
256                    MediaCodec::CONFIGURE_FLAG_ENCODE);
257    }
258
259    if (err != OK) {
260        return err;
261    }
262
263    err = mEncoder->start();
264
265    if (err != OK) {
266        return err;
267    }
268
269    err = mEncoder->getInputBuffers(&mEncoderInputBuffers);
270
271    if (err != OK) {
272        return err;
273    }
274
275    return mEncoder->getOutputBuffers(&mEncoderOutputBuffers);
276}
277
278void Converter::notifyError(status_t err) {
279    sp<AMessage> notify = mNotify->dup();
280    notify->setInt32("what", kWhatError);
281    notify->setInt32("err", err);
282    notify->post();
283}
284
285// static
286bool Converter::IsSilence(const sp<ABuffer> &accessUnit) {
287    const uint8_t *ptr = accessUnit->data();
288    const uint8_t *end = ptr + accessUnit->size();
289    while (ptr < end) {
290        if (*ptr != 0) {
291            return false;
292        }
293        ++ptr;
294    }
295
296    return true;
297}
298
299void Converter::onMessageReceived(const sp<AMessage> &msg) {
300    switch (msg->what()) {
301        case kWhatMediaPullerNotify:
302        {
303            int32_t what;
304            CHECK(msg->findInt32("what", &what));
305
306            if (!mIsPCMAudio && mEncoder == NULL) {
307                ALOGV("got msg '%s' after encoder shutdown.",
308                      msg->debugString().c_str());
309
310                if (what == MediaPuller::kWhatAccessUnit) {
311                    sp<ABuffer> accessUnit;
312                    CHECK(msg->findBuffer("accessUnit", &accessUnit));
313
314                    ReleaseMediaBufferReference(accessUnit);
315                }
316                break;
317            }
318
319            if (what == MediaPuller::kWhatEOS) {
320                mInputBufferQueue.push_back(NULL);
321
322                feedEncoderInputBuffers();
323
324                scheduleDoMoreWork();
325            } else {
326                CHECK_EQ(what, MediaPuller::kWhatAccessUnit);
327
328                sp<ABuffer> accessUnit;
329                CHECK(msg->findBuffer("accessUnit", &accessUnit));
330
331                if (mIsVideo && mNumFramesToDrop) {
332                    --mNumFramesToDrop;
333                    ALOGI("dropping frame.");
334                    ReleaseMediaBufferReference(accessUnit);
335                    break;
336                }
337
338#if 0
339                void *mbuf;
340                if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf)
341                        && mbuf != NULL) {
342                    ALOGI("queueing mbuf %p", mbuf);
343                }
344#endif
345
346#if ENABLE_SILENCE_DETECTION
347                if (!mIsVideo) {
348                    if (IsSilence(accessUnit)) {
349                        if (mInSilentMode) {
350                            break;
351                        }
352
353                        int64_t nowUs = ALooper::GetNowUs();
354
355                        if (mFirstSilentFrameUs < 0ll) {
356                            mFirstSilentFrameUs = nowUs;
357                        } else if (nowUs >= mFirstSilentFrameUs + 10000000ll) {
358                            mInSilentMode = true;
359                            ALOGI("audio in silent mode now.");
360                            break;
361                        }
362                    } else {
363                        if (mInSilentMode) {
364                            ALOGI("audio no longer in silent mode.");
365                        }
366                        mInSilentMode = false;
367                        mFirstSilentFrameUs = -1ll;
368                    }
369                }
370#endif
371
372                mInputBufferQueue.push_back(accessUnit);
373
374                feedEncoderInputBuffers();
375
376                scheduleDoMoreWork();
377            }
378            break;
379        }
380
381        case kWhatEncoderActivity:
382        {
383#if 0
384            int64_t whenUs;
385            if (msg->findInt64("whenUs", &whenUs)) {
386                int64_t nowUs = ALooper::GetNowUs();
387                ALOGI("[%s] kWhatEncoderActivity after %lld us",
388                      mIsVideo ? "video" : "audio", nowUs - whenUs);
389            }
390#endif
391
392            mDoMoreWorkPending = false;
393
394            if (mEncoder == NULL) {
395                break;
396            }
397
398            status_t err = doMoreWork();
399
400            if (err != OK) {
401                notifyError(err);
402            } else {
403                scheduleDoMoreWork();
404            }
405            break;
406        }
407
408        case kWhatRequestIDRFrame:
409        {
410            if (mEncoder == NULL) {
411                break;
412            }
413
414            if (mIsVideo) {
415                ALOGI("requesting IDR frame");
416                mEncoder->requestIDRFrame();
417            }
418            break;
419        }
420
421        case kWhatShutdown:
422        {
423            ALOGI("shutting down %s encoder", mIsVideo ? "video" : "audio");
424
425            releaseEncoder();
426
427            AString mime;
428            CHECK(mInputFormat->findString("mime", &mime));
429            ALOGI("encoder (%s) shut down.", mime.c_str());
430            break;
431        }
432
433        case kWhatDropAFrame:
434        {
435            ++mNumFramesToDrop;
436            break;
437        }
438
439        default:
440            TRESPASS();
441    }
442}
443
444void Converter::scheduleDoMoreWork() {
445    if (mIsPCMAudio) {
446        // There's no encoder involved in this case.
447        return;
448    }
449
450    if (mDoMoreWorkPending) {
451        return;
452    }
453
454    mDoMoreWorkPending = true;
455
456#if 1
457    if (mEncoderActivityNotify == NULL) {
458        mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, id());
459    }
460    mEncoder->requestActivityNotification(mEncoderActivityNotify->dup());
461#else
462    sp<AMessage> notify = new AMessage(kWhatEncoderActivity, id());
463    notify->setInt64("whenUs", ALooper::GetNowUs());
464    mEncoder->requestActivityNotification(notify);
465#endif
466}
467
468status_t Converter::feedRawAudioInputBuffers() {
469    // Split incoming PCM audio into buffers of 6 AUs of 80 audio frames each
470    // and add a 4 byte header according to the wifi display specs.
471
472    while (!mInputBufferQueue.empty()) {
473        sp<ABuffer> buffer = *mInputBufferQueue.begin();
474        mInputBufferQueue.erase(mInputBufferQueue.begin());
475
476        int16_t *ptr = (int16_t *)buffer->data();
477        int16_t *stop = (int16_t *)(buffer->data() + buffer->size());
478        while (ptr < stop) {
479            *ptr = htons(*ptr);
480            ++ptr;
481        }
482
483        static const size_t kFrameSize = 2 * sizeof(int16_t);  // stereo
484        static const size_t kFramesPerAU = 80;
485        static const size_t kNumAUsPerPESPacket = 6;
486
487        if (mPartialAudioAU != NULL) {
488            size_t bytesMissingForFullAU =
489                kNumAUsPerPESPacket * kFramesPerAU * kFrameSize
490                - mPartialAudioAU->size() + 4;
491
492            size_t copy = buffer->size();
493            if(copy > bytesMissingForFullAU) {
494                copy = bytesMissingForFullAU;
495            }
496
497            memcpy(mPartialAudioAU->data() + mPartialAudioAU->size(),
498                   buffer->data(),
499                   copy);
500
501            mPartialAudioAU->setRange(0, mPartialAudioAU->size() + copy);
502
503            buffer->setRange(buffer->offset() + copy, buffer->size() - copy);
504
505            int64_t timeUs;
506            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
507
508            int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0);
509            timeUs += copyUs;
510            buffer->meta()->setInt64("timeUs", timeUs);
511
512            if (bytesMissingForFullAU == copy) {
513                sp<AMessage> notify = mNotify->dup();
514                notify->setInt32("what", kWhatAccessUnit);
515                notify->setBuffer("accessUnit", mPartialAudioAU);
516                notify->post();
517
518                mPartialAudioAU.clear();
519            }
520        }
521
522        while (buffer->size() > 0) {
523            sp<ABuffer> partialAudioAU =
524                new ABuffer(
525                        4
526                        + kNumAUsPerPESPacket * kFrameSize * kFramesPerAU);
527
528            uint8_t *ptr = partialAudioAU->data();
529            ptr[0] = 0xa0;  // 10100000b
530            ptr[1] = kNumAUsPerPESPacket;
531            ptr[2] = 0;  // reserved, audio _emphasis_flag = 0
532
533            static const unsigned kQuantizationWordLength = 0;  // 16-bit
534            static const unsigned kAudioSamplingFrequency = 2;  // 48Khz
535            static const unsigned kNumberOfAudioChannels = 1;  // stereo
536
537            ptr[3] = (kQuantizationWordLength << 6)
538                    | (kAudioSamplingFrequency << 3)
539                    | kNumberOfAudioChannels;
540
541            size_t copy = buffer->size();
542            if (copy > partialAudioAU->size() - 4) {
543                copy = partialAudioAU->size() - 4;
544            }
545
546            memcpy(&ptr[4], buffer->data(), copy);
547
548            partialAudioAU->setRange(0, 4 + copy);
549            buffer->setRange(buffer->offset() + copy, buffer->size() - copy);
550
551            int64_t timeUs;
552            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
553
554            partialAudioAU->meta()->setInt64("timeUs", timeUs);
555
556            int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0);
557            timeUs += copyUs;
558            buffer->meta()->setInt64("timeUs", timeUs);
559
560            if (copy == partialAudioAU->capacity() - 4) {
561                sp<AMessage> notify = mNotify->dup();
562                notify->setInt32("what", kWhatAccessUnit);
563                notify->setBuffer("accessUnit", partialAudioAU);
564                notify->post();
565
566                partialAudioAU.clear();
567                continue;
568            }
569
570            mPartialAudioAU = partialAudioAU;
571        }
572    }
573
574    return OK;
575}
576
577status_t Converter::feedEncoderInputBuffers() {
578    if (mIsPCMAudio) {
579        return feedRawAudioInputBuffers();
580    }
581
582    while (!mInputBufferQueue.empty()
583            && !mAvailEncoderInputIndices.empty()) {
584        sp<ABuffer> buffer = *mInputBufferQueue.begin();
585        mInputBufferQueue.erase(mInputBufferQueue.begin());
586
587        size_t bufferIndex = *mAvailEncoderInputIndices.begin();
588        mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin());
589
590        int64_t timeUs = 0ll;
591        uint32_t flags = 0;
592
593        if (buffer != NULL) {
594            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
595
596            memcpy(mEncoderInputBuffers.itemAt(bufferIndex)->data(),
597                   buffer->data(),
598                   buffer->size());
599
600            void *mediaBuffer;
601            if (buffer->meta()->findPointer("mediaBuffer", &mediaBuffer)
602                    && mediaBuffer != NULL) {
603                mEncoderInputBuffers.itemAt(bufferIndex)->meta()
604                    ->setPointer("mediaBuffer", mediaBuffer);
605
606                buffer->meta()->setPointer("mediaBuffer", NULL);
607            }
608        } else {
609            flags = MediaCodec::BUFFER_FLAG_EOS;
610        }
611
612        status_t err = mEncoder->queueInputBuffer(
613                bufferIndex, 0, (buffer == NULL) ? 0 : buffer->size(),
614                timeUs, flags);
615
616        if (err != OK) {
617            return err;
618        }
619    }
620
621    return OK;
622}
623
624status_t Converter::doMoreWork() {
625#if 0
626    if (mIsVideo) {
627        int32_t videoBitrate = getBitrate("media.wfd.video-bitrate", 5000000);
628        if (videoBitrate != mPrevVideoBitrate) {
629            sp<AMessage> params = new AMessage;
630
631            params->setInt32("videoBitrate", videoBitrate);
632            mEncoder->setParameters(params);
633
634            mPrevVideoBitrate = videoBitrate;
635        }
636    }
637#endif
638
639    status_t err;
640
641    for (;;) {
642        size_t bufferIndex;
643        err = mEncoder->dequeueInputBuffer(&bufferIndex);
644
645        if (err != OK) {
646            break;
647        }
648
649        mAvailEncoderInputIndices.push_back(bufferIndex);
650    }
651
652    feedEncoderInputBuffers();
653
654    for (;;) {
655        size_t bufferIndex;
656        size_t offset;
657        size_t size;
658        int64_t timeUs;
659        uint32_t flags;
660        err = mEncoder->dequeueOutputBuffer(
661                &bufferIndex, &offset, &size, &timeUs, &flags);
662
663        if (err != OK) {
664            if (err == -EAGAIN) {
665                err = OK;
666            }
667            break;
668        }
669
670        if (flags & MediaCodec::BUFFER_FLAG_EOS) {
671            sp<AMessage> notify = mNotify->dup();
672            notify->setInt32("what", kWhatEOS);
673            notify->post();
674        } else {
675            sp<ABuffer> buffer = new ABuffer(size);
676            buffer->meta()->setInt64("timeUs", timeUs);
677
678            ALOGV("[%s] time %lld us (%.2f secs)",
679                  mIsVideo ? "video" : "audio", timeUs, timeUs / 1E6);
680
681            memcpy(buffer->data(),
682                   mEncoderOutputBuffers.itemAt(bufferIndex)->base() + offset,
683                   size);
684
685            if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) {
686                mOutputFormat->setBuffer("csd-0", buffer);
687            } else {
688                sp<AMessage> notify = mNotify->dup();
689                notify->setInt32("what", kWhatAccessUnit);
690                notify->setBuffer("accessUnit", buffer);
691                notify->post();
692            }
693        }
694
695        mEncoder->releaseOutputBuffer(bufferIndex);
696
697        if (flags & MediaCodec::BUFFER_FLAG_EOS) {
698            break;
699        }
700    }
701
702    return err;
703}
704
705void Converter::requestIDRFrame() {
706    (new AMessage(kWhatRequestIDRFrame, id()))->post();
707}
708
709void Converter::dropAFrame() {
710    (new AMessage(kWhatDropAFrame, id()))->post();
711}
712
713int32_t Converter::getVideoBitrate() const {
714    return mPrevVideoBitrate;
715}
716
717void Converter::setVideoBitrate(int32_t bitRate) {
718    if (mIsVideo && mEncoder != NULL && bitRate != mPrevVideoBitrate) {
719        sp<AMessage> params = new AMessage;
720        params->setInt32("videoBitrate", bitRate);
721
722        mEncoder->setParameters(params);
723
724        mPrevVideoBitrate = bitRate;
725    }
726}
727
728}  // namespace android
729