Converter.cpp revision 1a2952aee048ca7b1765e2bc09ebe9aeddaeafa3 
1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "Converter"
19#include <utils/Log.h>
20
21#include "Converter.h"
22
23#include "MediaPuller.h"
24
25#include <cutils/properties.h>
26#include <gui/Surface.h>
27#include <media/ICrypto.h>
28#include <media/stagefright/foundation/ABuffer.h>
29#include <media/stagefright/foundation/ADebug.h>
30#include <media/stagefright/foundation/AMessage.h>
31#include <media/stagefright/MediaBuffer.h>
32#include <media/stagefright/MediaCodec.h>
33#include <media/stagefright/MediaDefs.h>
34#include <media/stagefright/MediaErrors.h>
35
36#include <OMX_Video.h>
37
38namespace android {
39
40Converter::Converter(
41        const sp<AMessage> &notify,
42        const sp<ALooper> &codecLooper,
43        const sp<AMessage> &format,
44        bool usePCMAudio)
45    : mInitCheck(NO_INIT),
46      mNotify(notify),
47      mCodecLooper(codecLooper),
48      mInputFormat(format),
49      mIsVideo(false),
50      mIsPCMAudio(usePCMAudio),
51      mNeedToManuallyPrependSPSPPS(false),
52      mDoMoreWorkPending(false)
53#if ENABLE_SILENCE_DETECTION
54      ,mFirstSilentFrameUs(-1ll)
55      ,mInSilentMode(false)
56#endif
57      ,mPrevVideoBitrate(-1)
58    {
59    AString mime;
60    CHECK(mInputFormat->findString("mime", &mime));
61
62    if (!strncasecmp("video/", mime.c_str(), 6)) {
63        mIsVideo = true;
64    }
65
66    CHECK(!usePCMAudio || !mIsVideo);
67
68    mInitCheck = initEncoder();
69
70    if (mInitCheck != OK) {
71        releaseEncoder();
72    }
73}
74
75static void ReleaseMediaBufferReference(const sp<ABuffer> &accessUnit) {
76    void *mbuf;
77    if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf)
78            && mbuf != NULL) {
79        ALOGV("releasing mbuf %p", mbuf);
80
81        accessUnit->meta()->setPointer("mediaBuffer", NULL);
82
83        static_cast<MediaBuffer *>(mbuf)->release();
84        mbuf = NULL;
85    }
86}
87
88void Converter::releaseEncoder() {
89    if (mEncoder == NULL) {
90        return;
91    }
92
93    mEncoder->release();
94    mEncoder.clear();
95
96    while (!mInputBufferQueue.empty()) {
97        sp<ABuffer> accessUnit = *mInputBufferQueue.begin();
98        mInputBufferQueue.erase(mInputBufferQueue.begin());
99
100        ReleaseMediaBufferReference(accessUnit);
101    }
102
103    for (size_t i = 0; i < mEncoderInputBuffers.size(); ++i) {
104        sp<ABuffer> accessUnit = mEncoderInputBuffers.itemAt(i);
105        ReleaseMediaBufferReference(accessUnit);
106    }
107
108    mEncoderInputBuffers.clear();
109    mEncoderOutputBuffers.clear();
110}
111
112Converter::~Converter() {
113    CHECK(mEncoder == NULL);
114}
115
116void Converter::shutdownAsync() {
117    ALOGV("shutdown");
118    (new AMessage(kWhatShutdown, id()))->post();
119}
120
121status_t Converter::initCheck() const {
122    return mInitCheck;
123}
124
125size_t Converter::getInputBufferCount() const {
126    return mEncoderInputBuffers.size();
127}
128
129sp<AMessage> Converter::getOutputFormat() const {
130    return mOutputFormat;
131}
132
133bool Converter::needToManuallyPrependSPSPPS() const {
134    return mNeedToManuallyPrependSPSPPS;
135}
136
137static int32_t getBitrate(const char *propName, int32_t defaultValue) {
138    char val[PROPERTY_VALUE_MAX];
139    if (property_get(propName, val, NULL)) {
140        char *end;
141        unsigned long x = strtoul(val, &end, 10);
142
143        if (*end == '\0' && end > val && x > 0) {
144            return x;
145        }
146    }
147
148    return defaultValue;
149}
150
151status_t Converter::initEncoder() {
152    AString inputMIME;
153    CHECK(mInputFormat->findString("mime", &inputMIME));
154
155    AString outputMIME;
156    bool isAudio = false;
157    if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_AUDIO_RAW)) {
158        if (mIsPCMAudio) {
159            outputMIME = MEDIA_MIMETYPE_AUDIO_RAW;
160        } else {
161            outputMIME = MEDIA_MIMETYPE_AUDIO_AAC;
162        }
163        isAudio = true;
164    } else if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) {
165        outputMIME = MEDIA_MIMETYPE_VIDEO_AVC;
166    } else {
167        TRESPASS();
168    }
169
170    if (!mIsPCMAudio) {
171        mEncoder = MediaCodec::CreateByType(
172                mCodecLooper, outputMIME.c_str(), true /* encoder */);
173
174        if (mEncoder == NULL) {
175            return ERROR_UNSUPPORTED;
176        }
177    }
178
179    mOutputFormat = mInputFormat->dup();
180
181    if (mIsPCMAudio) {
182        return OK;
183    }
184
185    mOutputFormat->setString("mime", outputMIME.c_str());
186
187    int32_t audioBitrate = getBitrate("media.wfd.audio-bitrate", 128000);
188    int32_t videoBitrate = getBitrate("media.wfd.video-bitrate", 5000000);
189    mPrevVideoBitrate = videoBitrate;
190
191    ALOGI("using audio bitrate of %d bps, video bitrate of %d bps",
192          audioBitrate, videoBitrate);
193
194    if (isAudio) {
195        mOutputFormat->setInt32("bitrate", audioBitrate);
196    } else {
197        mOutputFormat->setInt32("bitrate", videoBitrate);
198        mOutputFormat->setInt32("bitrate-mode", OMX_Video_ControlRateConstant);
199        mOutputFormat->setInt32("frame-rate", 30);
200        mOutputFormat->setInt32("i-frame-interval", 15);  // Iframes every 15 secs
201
202        // Configure encoder to use intra macroblock refresh mode
203        mOutputFormat->setInt32("intra-refresh-mode", OMX_VIDEO_IntraRefreshCyclic);
204
205        int width, height, mbs;
206        if (!mOutputFormat->findInt32("width", &width)
207                || !mOutputFormat->findInt32("height", &height)) {
208            return ERROR_UNSUPPORTED;
209        }
210
211        // Update macroblocks in a cyclic fashion with 10% of all MBs within
212        // frame gets updated at one time. It takes about 10 frames to
213        // completely update a whole video frame. If the frame rate is 30,
214        // it takes about 333 ms in the best case (if next frame is not an IDR)
215        // to recover from a lost/corrupted packet.
216        mbs = (((width + 15) / 16) * ((height + 15) / 16) * 10) / 100;
217        mOutputFormat->setInt32("intra-refresh-CIR-mbs", mbs);
218    }
219
220    ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str());
221
222    mNeedToManuallyPrependSPSPPS = false;
223
224    status_t err = NO_INIT;
225
226    if (!isAudio) {
227        sp<AMessage> tmp = mOutputFormat->dup();
228        tmp->setInt32("prepend-sps-pps-to-idr-frames", 1);
229
230        err = mEncoder->configure(
231                tmp,
232                NULL /* nativeWindow */,
233                NULL /* crypto */,
234                MediaCodec::CONFIGURE_FLAG_ENCODE);
235
236        if (err == OK) {
237            // Encoder supported prepending SPS/PPS, we don't need to emulate
238            // it.
239            mOutputFormat = tmp;
240        } else {
241            mNeedToManuallyPrependSPSPPS = true;
242
243            ALOGI("We going to manually prepend SPS and PPS to IDR frames.");
244        }
245    }
246
247    if (err != OK) {
248        // We'll get here for audio or if we failed to configure the encoder
249        // to automatically prepend SPS/PPS in the case of video.
250
251        err = mEncoder->configure(
252                    mOutputFormat,
253                    NULL /* nativeWindow */,
254                    NULL /* crypto */,
255                    MediaCodec::CONFIGURE_FLAG_ENCODE);
256    }
257
258    if (err != OK) {
259        return err;
260    }
261
262    err = mEncoder->start();
263
264    if (err != OK) {
265        return err;
266    }
267
268    err = mEncoder->getInputBuffers(&mEncoderInputBuffers);
269
270    if (err != OK) {
271        return err;
272    }
273
274    return mEncoder->getOutputBuffers(&mEncoderOutputBuffers);
275}
276
277void Converter::notifyError(status_t err) {
278    sp<AMessage> notify = mNotify->dup();
279    notify->setInt32("what", kWhatError);
280    notify->setInt32("err", err);
281    notify->post();
282}
283
284// static
285bool Converter::IsSilence(const sp<ABuffer> &accessUnit) {
286    const uint8_t *ptr = accessUnit->data();
287    const uint8_t *end = ptr + accessUnit->size();
288    while (ptr < end) {
289        if (*ptr != 0) {
290            return false;
291        }
292        ++ptr;
293    }
294
295    return true;
296}
297
298void Converter::onMessageReceived(const sp<AMessage> &msg) {
299    switch (msg->what()) {
300        case kWhatMediaPullerNotify:
301        {
302            int32_t what;
303            CHECK(msg->findInt32("what", &what));
304
305            if (!mIsPCMAudio && mEncoder == NULL) {
306                ALOGV("got msg '%s' after encoder shutdown.",
307                      msg->debugString().c_str());
308
309                if (what == MediaPuller::kWhatAccessUnit) {
310                    sp<ABuffer> accessUnit;
311                    CHECK(msg->findBuffer("accessUnit", &accessUnit));
312
313                    ReleaseMediaBufferReference(accessUnit);
314                }
315                break;
316            }
317
318            if (what == MediaPuller::kWhatEOS) {
319                mInputBufferQueue.push_back(NULL);
320
321                feedEncoderInputBuffers();
322
323                scheduleDoMoreWork();
324            } else {
325                CHECK_EQ(what, MediaPuller::kWhatAccessUnit);
326
327                sp<ABuffer> accessUnit;
328                CHECK(msg->findBuffer("accessUnit", &accessUnit));
329
330#if 0
331                void *mbuf;
332                if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf)
333                        && mbuf != NULL) {
334                    ALOGI("queueing mbuf %p", mbuf);
335                }
336#endif
337
338#if ENABLE_SILENCE_DETECTION
339                if (!mIsVideo) {
340                    if (IsSilence(accessUnit)) {
341                        if (mInSilentMode) {
342                            break;
343                        }
344
345                        int64_t nowUs = ALooper::GetNowUs();
346
347                        if (mFirstSilentFrameUs < 0ll) {
348                            mFirstSilentFrameUs = nowUs;
349                        } else if (nowUs >= mFirstSilentFrameUs + 10000000ll) {
350                            mInSilentMode = true;
351                            ALOGI("audio in silent mode now.");
352                            break;
353                        }
354                    } else {
355                        if (mInSilentMode) {
356                            ALOGI("audio no longer in silent mode.");
357                        }
358                        mInSilentMode = false;
359                        mFirstSilentFrameUs = -1ll;
360                    }
361                }
362#endif
363
364                mInputBufferQueue.push_back(accessUnit);
365
366                feedEncoderInputBuffers();
367
368                scheduleDoMoreWork();
369            }
370            break;
371        }
372
373        case kWhatEncoderActivity:
374        {
375#if 0
376            int64_t whenUs;
377            if (msg->findInt64("whenUs", &whenUs)) {
378                int64_t nowUs = ALooper::GetNowUs();
379                ALOGI("[%s] kWhatEncoderActivity after %lld us",
380                      mIsVideo ? "video" : "audio", nowUs - whenUs);
381            }
382#endif
383
384            mDoMoreWorkPending = false;
385
386            if (mEncoder == NULL) {
387                break;
388            }
389
390            status_t err = doMoreWork();
391
392            if (err != OK) {
393                notifyError(err);
394            } else {
395                scheduleDoMoreWork();
396            }
397            break;
398        }
399
400        case kWhatRequestIDRFrame:
401        {
402            if (mEncoder == NULL) {
403                break;
404            }
405
406            if (mIsVideo) {
407                ALOGI("requesting IDR frame");
408                mEncoder->requestIDRFrame();
409            }
410            break;
411        }
412
413        case kWhatShutdown:
414        {
415            ALOGI("shutting down %s encoder", mIsVideo ? "video" : "audio");
416
417            releaseEncoder();
418
419            AString mime;
420            CHECK(mInputFormat->findString("mime", &mime));
421            ALOGI("encoder (%s) shut down.", mime.c_str());
422            break;
423        }
424
425        default:
426            TRESPASS();
427    }
428}
429
430void Converter::scheduleDoMoreWork() {
431    if (mIsPCMAudio) {
432        // There's no encoder involved in this case.
433        return;
434    }
435
436    if (mDoMoreWorkPending) {
437        return;
438    }
439
440    mDoMoreWorkPending = true;
441
442#if 1
443    if (mEncoderActivityNotify == NULL) {
444        mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, id());
445    }
446    mEncoder->requestActivityNotification(mEncoderActivityNotify->dup());
447#else
448    sp<AMessage> notify = new AMessage(kWhatEncoderActivity, id());
449    notify->setInt64("whenUs", ALooper::GetNowUs());
450    mEncoder->requestActivityNotification(notify);
451#endif
452}
453
454status_t Converter::feedRawAudioInputBuffers() {
455    // Split incoming PCM audio into buffers of 6 AUs of 80 audio frames each
456    // and add a 4 byte header according to the wifi display specs.
457
458    while (!mInputBufferQueue.empty()) {
459        sp<ABuffer> buffer = *mInputBufferQueue.begin();
460        mInputBufferQueue.erase(mInputBufferQueue.begin());
461
462        int16_t *ptr = (int16_t *)buffer->data();
463        int16_t *stop = (int16_t *)(buffer->data() + buffer->size());
464        while (ptr < stop) {
465            *ptr = htons(*ptr);
466            ++ptr;
467        }
468
469        static const size_t kFrameSize = 2 * sizeof(int16_t);  // stereo
470        static const size_t kFramesPerAU = 80;
471        static const size_t kNumAUsPerPESPacket = 6;
472
473        if (mPartialAudioAU != NULL) {
474            size_t bytesMissingForFullAU =
475                kNumAUsPerPESPacket * kFramesPerAU * kFrameSize
476                - mPartialAudioAU->size() + 4;
477
478            size_t copy = buffer->size();
479            if(copy > bytesMissingForFullAU) {
480                copy = bytesMissingForFullAU;
481            }
482
483            memcpy(mPartialAudioAU->data() + mPartialAudioAU->size(),
484                   buffer->data(),
485                   copy);
486
487            mPartialAudioAU->setRange(0, mPartialAudioAU->size() + copy);
488
489            buffer->setRange(buffer->offset() + copy, buffer->size() - copy);
490
491            int64_t timeUs;
492            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
493
494            int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0);
495            timeUs += copyUs;
496            buffer->meta()->setInt64("timeUs", timeUs);
497
498            if (bytesMissingForFullAU == copy) {
499                sp<AMessage> notify = mNotify->dup();
500                notify->setInt32("what", kWhatAccessUnit);
501                notify->setBuffer("accessUnit", mPartialAudioAU);
502                notify->post();
503
504                mPartialAudioAU.clear();
505            }
506        }
507
508        while (buffer->size() > 0) {
509            sp<ABuffer> partialAudioAU =
510                new ABuffer(
511                        4
512                        + kNumAUsPerPESPacket * kFrameSize * kFramesPerAU);
513
514            uint8_t *ptr = partialAudioAU->data();
515            ptr[0] = 0xa0;  // 10100000b
516            ptr[1] = kNumAUsPerPESPacket;
517            ptr[2] = 0;  // reserved, audio _emphasis_flag = 0
518
519            static const unsigned kQuantizationWordLength = 0;  // 16-bit
520            static const unsigned kAudioSamplingFrequency = 2;  // 48Khz
521            static const unsigned kNumberOfAudioChannels = 1;  // stereo
522
523            ptr[3] = (kQuantizationWordLength << 6)
524                    | (kAudioSamplingFrequency << 3)
525                    | kNumberOfAudioChannels;
526
527            size_t copy = buffer->size();
528            if (copy > partialAudioAU->size() - 4) {
529                copy = partialAudioAU->size() - 4;
530            }
531
532            memcpy(&ptr[4], buffer->data(), copy);
533
534            partialAudioAU->setRange(0, 4 + copy);
535            buffer->setRange(buffer->offset() + copy, buffer->size() - copy);
536
537            int64_t timeUs;
538            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
539
540            partialAudioAU->meta()->setInt64("timeUs", timeUs);
541
542            int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0);
543            timeUs += copyUs;
544            buffer->meta()->setInt64("timeUs", timeUs);
545
546            if (copy == partialAudioAU->capacity() - 4) {
547                sp<AMessage> notify = mNotify->dup();
548                notify->setInt32("what", kWhatAccessUnit);
549                notify->setBuffer("accessUnit", partialAudioAU);
550                notify->post();
551
552                partialAudioAU.clear();
553                continue;
554            }
555
556            mPartialAudioAU = partialAudioAU;
557        }
558    }
559
560    return OK;
561}
562
563status_t Converter::feedEncoderInputBuffers() {
564    if (mIsPCMAudio) {
565        return feedRawAudioInputBuffers();
566    }
567
568    while (!mInputBufferQueue.empty()
569            && !mAvailEncoderInputIndices.empty()) {
570        sp<ABuffer> buffer = *mInputBufferQueue.begin();
571        mInputBufferQueue.erase(mInputBufferQueue.begin());
572
573        size_t bufferIndex = *mAvailEncoderInputIndices.begin();
574        mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin());
575
576        int64_t timeUs = 0ll;
577        uint32_t flags = 0;
578
579        if (buffer != NULL) {
580            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
581
582            memcpy(mEncoderInputBuffers.itemAt(bufferIndex)->data(),
583                   buffer->data(),
584                   buffer->size());
585
586            void *mediaBuffer;
587            if (buffer->meta()->findPointer("mediaBuffer", &mediaBuffer)
588                    && mediaBuffer != NULL) {
589                mEncoderInputBuffers.itemAt(bufferIndex)->meta()
590                    ->setPointer("mediaBuffer", mediaBuffer);
591
592                buffer->meta()->setPointer("mediaBuffer", NULL);
593            }
594        } else {
595            flags = MediaCodec::BUFFER_FLAG_EOS;
596        }
597
598        status_t err = mEncoder->queueInputBuffer(
599                bufferIndex, 0, (buffer == NULL) ? 0 : buffer->size(),
600                timeUs, flags);
601
602        if (err != OK) {
603            return err;
604        }
605    }
606
607    return OK;
608}
609
610status_t Converter::doMoreWork() {
611    if (mIsVideo) {
612        int32_t videoBitrate = getBitrate("media.wfd.video-bitrate", 5000000);
613        if (videoBitrate != mPrevVideoBitrate) {
614            sp<AMessage> params = new AMessage;
615
616            params->setInt32("videoBitrate", videoBitrate);
617            mEncoder->setParameters(params);
618
619            mPrevVideoBitrate = videoBitrate;
620        }
621    }
622
623    status_t err;
624
625    for (;;) {
626        size_t bufferIndex;
627        err = mEncoder->dequeueInputBuffer(&bufferIndex);
628
629        if (err != OK) {
630            break;
631        }
632
633        mAvailEncoderInputIndices.push_back(bufferIndex);
634    }
635
636    feedEncoderInputBuffers();
637
638    for (;;) {
639        size_t bufferIndex;
640        size_t offset;
641        size_t size;
642        int64_t timeUs;
643        uint32_t flags;
644        err = mEncoder->dequeueOutputBuffer(
645                &bufferIndex, &offset, &size, &timeUs, &flags);
646
647        if (err != OK) {
648            if (err == -EAGAIN) {
649                err = OK;
650            }
651            break;
652        }
653
654        if (flags & MediaCodec::BUFFER_FLAG_EOS) {
655            sp<AMessage> notify = mNotify->dup();
656            notify->setInt32("what", kWhatEOS);
657            notify->post();
658        } else {
659            sp<ABuffer> buffer = new ABuffer(size);
660            buffer->meta()->setInt64("timeUs", timeUs);
661
662            ALOGV("[%s] time %lld us (%.2f secs)",
663                  mIsVideo ? "video" : "audio", timeUs, timeUs / 1E6);
664
665            memcpy(buffer->data(),
666                   mEncoderOutputBuffers.itemAt(bufferIndex)->base() + offset,
667                   size);
668
669            if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) {
670                mOutputFormat->setBuffer("csd-0", buffer);
671            } else {
672                sp<AMessage> notify = mNotify->dup();
673                notify->setInt32("what", kWhatAccessUnit);
674                notify->setBuffer("accessUnit", buffer);
675                notify->post();
676            }
677        }
678
679        mEncoder->releaseOutputBuffer(bufferIndex);
680
681        if (flags & MediaCodec::BUFFER_FLAG_EOS) {
682            break;
683        }
684    }
685
686    return err;
687}
688
689void Converter::requestIDRFrame() {
690    (new AMessage(kWhatRequestIDRFrame, id()))->post();
691}
692
693}  // namespace android
694