Converter.cpp revision 96076964863ee6887f0bed9d0f11f424b48ab9b9 
1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "Converter"
19#include <utils/Log.h>
20
21#include "Converter.h"
22
23#include "MediaPuller.h"
24
25#include <cutils/properties.h>
26#include <gui/SurfaceTextureClient.h>
27#include <media/ICrypto.h>
28#include <media/stagefright/foundation/ABuffer.h>
29#include <media/stagefright/foundation/ADebug.h>
30#include <media/stagefright/foundation/AMessage.h>
31#include <media/stagefright/MediaBuffer.h>
32#include <media/stagefright/MediaCodec.h>
33#include <media/stagefright/MediaDefs.h>
34#include <media/stagefright/MediaErrors.h>
35
36#include <OMX_Video.h>
37
38namespace android {
39
40Converter::Converter(
41        const sp<AMessage> &notify,
42        const sp<ALooper> &codecLooper,
43        const sp<AMessage> &format,
44        bool usePCMAudio)
45    : mInitCheck(NO_INIT),
46      mNotify(notify),
47      mCodecLooper(codecLooper),
48      mInputFormat(format),
49      mIsVideo(false),
50      mIsPCMAudio(usePCMAudio),
51      mDoMoreWorkPending(false)
52#if ENABLE_SILENCE_DETECTION
53      ,mFirstSilentFrameUs(-1ll)
54      ,mInSilentMode(false)
55#endif
56    {
57    AString mime;
58    CHECK(mInputFormat->findString("mime", &mime));
59
60    if (!strncasecmp("video/", mime.c_str(), 6)) {
61        mIsVideo = true;
62    }
63
64    CHECK(!usePCMAudio || !mIsVideo);
65
66    mInitCheck = initEncoder();
67
68    if (mInitCheck != OK) {
69        if (mEncoder != NULL) {
70            mEncoder->release();
71            mEncoder.clear();
72        }
73    }
74}
75
76Converter::~Converter() {
77    CHECK(mEncoder == NULL);
78}
79
80void Converter::shutdownAsync() {
81    ALOGV("shutdown");
82    (new AMessage(kWhatShutdown, id()))->post();
83}
84
85status_t Converter::initCheck() const {
86    return mInitCheck;
87}
88
89size_t Converter::getInputBufferCount() const {
90    return mEncoderInputBuffers.size();
91}
92
93sp<AMessage> Converter::getOutputFormat() const {
94    return mOutputFormat;
95}
96
97static int32_t getBitrate(const char *propName, int32_t defaultValue) {
98    char val[PROPERTY_VALUE_MAX];
99    if (property_get(propName, val, NULL)) {
100        char *end;
101        unsigned long x = strtoul(val, &end, 10);
102
103        if (*end == '\0' && end > val && x > 0) {
104            return x;
105        }
106    }
107
108    return defaultValue;
109}
110
111status_t Converter::initEncoder() {
112    AString inputMIME;
113    CHECK(mInputFormat->findString("mime", &inputMIME));
114
115    AString outputMIME;
116    bool isAudio = false;
117    if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_AUDIO_RAW)) {
118        if (mIsPCMAudio) {
119            outputMIME = MEDIA_MIMETYPE_AUDIO_RAW;
120        } else {
121            outputMIME = MEDIA_MIMETYPE_AUDIO_AAC;
122        }
123        isAudio = true;
124    } else if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) {
125        outputMIME = MEDIA_MIMETYPE_VIDEO_AVC;
126    } else {
127        TRESPASS();
128    }
129
130    if (!mIsPCMAudio) {
131        mEncoder = MediaCodec::CreateByType(
132                mCodecLooper, outputMIME.c_str(), true /* encoder */);
133
134        if (mEncoder == NULL) {
135            return ERROR_UNSUPPORTED;
136        }
137    }
138
139    mOutputFormat = mInputFormat->dup();
140
141    if (mIsPCMAudio) {
142        return OK;
143    }
144
145    mOutputFormat->setString("mime", outputMIME.c_str());
146
147    int32_t audioBitrate = getBitrate("media.wfd.audio-bitrate", 128000);
148    int32_t videoBitrate = getBitrate("media.wfd.video-bitrate", 5000000);
149
150    ALOGI("using audio bitrate of %d bps, video bitrate of %d bps",
151          audioBitrate, videoBitrate);
152
153    if (isAudio) {
154        mOutputFormat->setInt32("bitrate", audioBitrate);
155    } else {
156        mOutputFormat->setInt32("bitrate", videoBitrate);
157        mOutputFormat->setInt32("bitrate-mode", OMX_Video_ControlRateConstant);
158        mOutputFormat->setInt32("frame-rate", 30);
159        mOutputFormat->setInt32("i-frame-interval", 1);  // Iframes every 1 secs
160        mOutputFormat->setInt32("prepend-sps-pps-to-idr-frames", 1);
161    }
162
163    ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str());
164
165    status_t err = mEncoder->configure(
166            mOutputFormat,
167            NULL /* nativeWindow */,
168            NULL /* crypto */,
169            MediaCodec::CONFIGURE_FLAG_ENCODE);
170
171    if (err != OK) {
172        return err;
173    }
174
175    err = mEncoder->start();
176
177    if (err != OK) {
178        return err;
179    }
180
181    err = mEncoder->getInputBuffers(&mEncoderInputBuffers);
182
183    if (err != OK) {
184        return err;
185    }
186
187    return mEncoder->getOutputBuffers(&mEncoderOutputBuffers);
188}
189
190void Converter::notifyError(status_t err) {
191    sp<AMessage> notify = mNotify->dup();
192    notify->setInt32("what", kWhatError);
193    notify->setInt32("err", err);
194    notify->post();
195}
196
197// static
198bool Converter::IsSilence(const sp<ABuffer> &accessUnit) {
199    const uint8_t *ptr = accessUnit->data();
200    const uint8_t *end = ptr + accessUnit->size();
201    while (ptr < end) {
202        if (*ptr != 0) {
203            return false;
204        }
205        ++ptr;
206    }
207
208    return true;
209}
210
211void Converter::onMessageReceived(const sp<AMessage> &msg) {
212    switch (msg->what()) {
213        case kWhatMediaPullerNotify:
214        {
215            int32_t what;
216            CHECK(msg->findInt32("what", &what));
217
218            if (!mIsPCMAudio && mEncoder == NULL) {
219                ALOGV("got msg '%s' after encoder shutdown.",
220                      msg->debugString().c_str());
221
222                if (what == MediaPuller::kWhatAccessUnit) {
223                    sp<ABuffer> accessUnit;
224                    CHECK(msg->findBuffer("accessUnit", &accessUnit));
225
226                    void *mbuf;
227                    if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf)
228                            && mbuf != NULL) {
229                        ALOGV("releasing mbuf %p", mbuf);
230
231                        accessUnit->meta()->setPointer("mediaBuffer", NULL);
232
233                        static_cast<MediaBuffer *>(mbuf)->release();
234                        mbuf = NULL;
235                    }
236                }
237                break;
238            }
239
240            if (what == MediaPuller::kWhatEOS) {
241                mInputBufferQueue.push_back(NULL);
242
243                feedEncoderInputBuffers();
244
245                scheduleDoMoreWork();
246            } else {
247                CHECK_EQ(what, MediaPuller::kWhatAccessUnit);
248
249                sp<ABuffer> accessUnit;
250                CHECK(msg->findBuffer("accessUnit", &accessUnit));
251
252#if 0
253                void *mbuf;
254                if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf)
255                        && mbuf != NULL) {
256                    ALOGI("queueing mbuf %p", mbuf);
257                }
258#endif
259
260#if ENABLE_SILENCE_DETECTION
261                if (!mIsVideo) {
262                    if (IsSilence(accessUnit)) {
263                        if (mInSilentMode) {
264                            break;
265                        }
266
267                        int64_t nowUs = ALooper::GetNowUs();
268
269                        if (mFirstSilentFrameUs < 0ll) {
270                            mFirstSilentFrameUs = nowUs;
271                        } else if (nowUs >= mFirstSilentFrameUs + 10000000ll) {
272                            mInSilentMode = true;
273                            ALOGI("audio in silent mode now.");
274                            break;
275                        }
276                    } else {
277                        if (mInSilentMode) {
278                            ALOGI("audio no longer in silent mode.");
279                        }
280                        mInSilentMode = false;
281                        mFirstSilentFrameUs = -1ll;
282                    }
283                }
284#endif
285
286                mInputBufferQueue.push_back(accessUnit);
287
288                feedEncoderInputBuffers();
289
290                scheduleDoMoreWork();
291            }
292            break;
293        }
294
295        case kWhatEncoderActivity:
296        {
297#if 0
298            int64_t whenUs;
299            if (msg->findInt64("whenUs", &whenUs)) {
300                int64_t nowUs = ALooper::GetNowUs();
301                ALOGI("[%s] kWhatEncoderActivity after %lld us",
302                      mIsVideo ? "video" : "audio", nowUs - whenUs);
303            }
304#endif
305
306            mDoMoreWorkPending = false;
307
308            if (mEncoder == NULL) {
309                break;
310            }
311
312            status_t err = doMoreWork();
313
314            if (err != OK) {
315                notifyError(err);
316            } else {
317                scheduleDoMoreWork();
318            }
319            break;
320        }
321
322        case kWhatRequestIDRFrame:
323        {
324            if (mEncoder == NULL) {
325                break;
326            }
327
328            if (mIsVideo) {
329                ALOGI("requesting IDR frame");
330                mEncoder->requestIDRFrame();
331            }
332            break;
333        }
334
335        case kWhatShutdown:
336        {
337            ALOGI("shutting down encoder");
338
339            if (mEncoder != NULL) {
340                mEncoder->release();
341                mEncoder.clear();
342            }
343
344            AString mime;
345            CHECK(mInputFormat->findString("mime", &mime));
346            ALOGI("encoder (%s) shut down.", mime.c_str());
347            break;
348        }
349
350        default:
351            TRESPASS();
352    }
353}
354
355void Converter::scheduleDoMoreWork() {
356    if (mIsPCMAudio) {
357        // There's no encoder involved in this case.
358        return;
359    }
360
361    if (mDoMoreWorkPending) {
362        return;
363    }
364
365    mDoMoreWorkPending = true;
366
367#if 1
368    if (mEncoderActivityNotify == NULL) {
369        mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, id());
370    }
371    mEncoder->requestActivityNotification(mEncoderActivityNotify->dup());
372#else
373    sp<AMessage> notify = new AMessage(kWhatEncoderActivity, id());
374    notify->setInt64("whenUs", ALooper::GetNowUs());
375    mEncoder->requestActivityNotification(notify);
376#endif
377}
378
379status_t Converter::feedRawAudioInputBuffers() {
380    // Split incoming PCM audio into buffers of 6 AUs of 80 audio frames each
381    // and add a 4 byte header according to the wifi display specs.
382
383    while (!mInputBufferQueue.empty()) {
384        sp<ABuffer> buffer = *mInputBufferQueue.begin();
385        mInputBufferQueue.erase(mInputBufferQueue.begin());
386
387        int16_t *ptr = (int16_t *)buffer->data();
388        int16_t *stop = (int16_t *)(buffer->data() + buffer->size());
389        while (ptr < stop) {
390            *ptr = htons(*ptr);
391            ++ptr;
392        }
393
394        static const size_t kFrameSize = 2 * sizeof(int16_t);  // stereo
395        static const size_t kFramesPerAU = 80;
396        static const size_t kNumAUsPerPESPacket = 6;
397
398        if (mPartialAudioAU != NULL) {
399            size_t bytesMissingForFullAU =
400                kNumAUsPerPESPacket * kFramesPerAU * kFrameSize
401                - mPartialAudioAU->size() + 4;
402
403            size_t copy = buffer->size();
404            if(copy > bytesMissingForFullAU) {
405                copy = bytesMissingForFullAU;
406            }
407
408            memcpy(mPartialAudioAU->data() + mPartialAudioAU->size(),
409                   buffer->data(),
410                   copy);
411
412            mPartialAudioAU->setRange(0, mPartialAudioAU->size() + copy);
413
414            buffer->setRange(buffer->offset() + copy, buffer->size() - copy);
415
416            int64_t timeUs;
417            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
418
419            int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0);
420            timeUs += copyUs;
421            buffer->meta()->setInt64("timeUs", timeUs);
422
423            if (bytesMissingForFullAU == copy) {
424                sp<AMessage> notify = mNotify->dup();
425                notify->setInt32("what", kWhatAccessUnit);
426                notify->setBuffer("accessUnit", mPartialAudioAU);
427                notify->post();
428
429                mPartialAudioAU.clear();
430            }
431        }
432
433        while (buffer->size() > 0) {
434            sp<ABuffer> partialAudioAU =
435                new ABuffer(
436                        4
437                        + kNumAUsPerPESPacket * kFrameSize * kFramesPerAU);
438
439            uint8_t *ptr = partialAudioAU->data();
440            ptr[0] = 0xa0;  // 10100000b
441            ptr[1] = kNumAUsPerPESPacket;
442            ptr[2] = 0;  // reserved, audio _emphasis_flag = 0
443
444            static const unsigned kQuantizationWordLength = 0;  // 16-bit
445            static const unsigned kAudioSamplingFrequency = 2;  // 48Khz
446            static const unsigned kNumberOfAudioChannels = 1;  // stereo
447
448            ptr[3] = (kQuantizationWordLength << 6)
449                    | (kAudioSamplingFrequency << 3)
450                    | kNumberOfAudioChannels;
451
452            size_t copy = buffer->size();
453            if (copy > partialAudioAU->size() - 4) {
454                copy = partialAudioAU->size() - 4;
455            }
456
457            memcpy(&ptr[4], buffer->data(), copy);
458
459            partialAudioAU->setRange(0, 4 + copy);
460            buffer->setRange(buffer->offset() + copy, buffer->size() - copy);
461
462            int64_t timeUs;
463            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
464
465            partialAudioAU->meta()->setInt64("timeUs", timeUs);
466
467            int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0);
468            timeUs += copyUs;
469            buffer->meta()->setInt64("timeUs", timeUs);
470
471            if (copy == partialAudioAU->capacity() - 4) {
472                sp<AMessage> notify = mNotify->dup();
473                notify->setInt32("what", kWhatAccessUnit);
474                notify->setBuffer("accessUnit", partialAudioAU);
475                notify->post();
476
477                partialAudioAU.clear();
478                continue;
479            }
480
481            mPartialAudioAU = partialAudioAU;
482        }
483    }
484
485    return OK;
486}
487
488status_t Converter::feedEncoderInputBuffers() {
489    if (mIsPCMAudio) {
490        return feedRawAudioInputBuffers();
491    }
492
493    while (!mInputBufferQueue.empty()
494            && !mAvailEncoderInputIndices.empty()) {
495        sp<ABuffer> buffer = *mInputBufferQueue.begin();
496        mInputBufferQueue.erase(mInputBufferQueue.begin());
497
498        size_t bufferIndex = *mAvailEncoderInputIndices.begin();
499        mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin());
500
501        int64_t timeUs = 0ll;
502        uint32_t flags = 0;
503
504        if (buffer != NULL) {
505            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
506
507            memcpy(mEncoderInputBuffers.itemAt(bufferIndex)->data(),
508                   buffer->data(),
509                   buffer->size());
510
511            void *mediaBuffer;
512            if (buffer->meta()->findPointer("mediaBuffer", &mediaBuffer)
513                    && mediaBuffer != NULL) {
514                mEncoderInputBuffers.itemAt(bufferIndex)->meta()
515                    ->setPointer("mediaBuffer", mediaBuffer);
516
517                buffer->meta()->setPointer("mediaBuffer", NULL);
518            }
519        } else {
520            flags = MediaCodec::BUFFER_FLAG_EOS;
521        }
522
523        status_t err = mEncoder->queueInputBuffer(
524                bufferIndex, 0, (buffer == NULL) ? 0 : buffer->size(),
525                timeUs, flags);
526
527        if (err != OK) {
528            return err;
529        }
530    }
531
532    return OK;
533}
534
535status_t Converter::doMoreWork() {
536    status_t err;
537
538    for (;;) {
539        size_t bufferIndex;
540        err = mEncoder->dequeueInputBuffer(&bufferIndex);
541
542        if (err != OK) {
543            break;
544        }
545
546        mAvailEncoderInputIndices.push_back(bufferIndex);
547    }
548
549    feedEncoderInputBuffers();
550
551    for (;;) {
552        size_t bufferIndex;
553        size_t offset;
554        size_t size;
555        int64_t timeUs;
556        uint32_t flags;
557        err = mEncoder->dequeueOutputBuffer(
558                &bufferIndex, &offset, &size, &timeUs, &flags);
559
560        if (err != OK) {
561            if (err == -EAGAIN) {
562                err = OK;
563            }
564            break;
565        }
566
567        if (flags & MediaCodec::BUFFER_FLAG_EOS) {
568            sp<AMessage> notify = mNotify->dup();
569            notify->setInt32("what", kWhatEOS);
570            notify->post();
571        } else {
572            sp<ABuffer> buffer = new ABuffer(size);
573            buffer->meta()->setInt64("timeUs", timeUs);
574
575            ALOGV("[%s] time %lld us (%.2f secs)",
576                  mIsVideo ? "video" : "audio", timeUs, timeUs / 1E6);
577
578            memcpy(buffer->data(),
579                   mEncoderOutputBuffers.itemAt(bufferIndex)->base() + offset,
580                   size);
581
582            if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) {
583                mOutputFormat->setBuffer("csd-0", buffer);
584            } else {
585                sp<AMessage> notify = mNotify->dup();
586                notify->setInt32("what", kWhatAccessUnit);
587                notify->setBuffer("accessUnit", buffer);
588                notify->post();
589            }
590        }
591
592        mEncoder->releaseOutputBuffer(bufferIndex);
593
594        if (flags & MediaCodec::BUFFER_FLAG_EOS) {
595            break;
596        }
597    }
598
599    return err;
600}
601
602void Converter::requestIDRFrame() {
603    (new AMessage(kWhatRequestIDRFrame, id()))->post();
604}
605
606}  // namespace android
607