1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "Converter"
19#include <utils/Log.h>
20
21#include "Converter.h"
22
23#include "MediaPuller.h"
24
25#include <cutils/properties.h>
26#include <gui/SurfaceTextureClient.h>
27#include <media/ICrypto.h>
28#include <media/stagefright/foundation/ABuffer.h>
29#include <media/stagefright/foundation/ADebug.h>
30#include <media/stagefright/foundation/AMessage.h>
31#include <media/stagefright/MediaBuffer.h>
32#include <media/stagefright/MediaCodec.h>
33#include <media/stagefright/MediaDefs.h>
34#include <media/stagefright/MediaErrors.h>
35
36#include <OMX_Video.h>
37
38namespace android {
39
40Converter::Converter(
41        const sp<AMessage> &notify,
42        const sp<ALooper> &codecLooper,
43        const sp<AMessage> &format,
44        bool usePCMAudio)
45    : mInitCheck(NO_INIT),
46      mNotify(notify),
47      mCodecLooper(codecLooper),
48      mInputFormat(format),
49      mIsVideo(false),
50      mIsPCMAudio(usePCMAudio),
51      mNeedToManuallyPrependSPSPPS(false),
52      mDoMoreWorkPending(false)
53#if ENABLE_SILENCE_DETECTION
54      ,mFirstSilentFrameUs(-1ll)
55      ,mInSilentMode(false)
56#endif
57    {
58    AString mime;
59    CHECK(mInputFormat->findString("mime", &mime));
60
61    if (!strncasecmp("video/", mime.c_str(), 6)) {
62        mIsVideo = true;
63    }
64
65    CHECK(!usePCMAudio || !mIsVideo);
66
67    mInitCheck = initEncoder();
68
69    if (mInitCheck != OK) {
70        if (mEncoder != NULL) {
71            mEncoder->release();
72            mEncoder.clear();
73        }
74    }
75}
76
77Converter::~Converter() {
78    CHECK(mEncoder == NULL);
79}
80
81void Converter::shutdownAsync() {
82    ALOGV("shutdown");
83    (new AMessage(kWhatShutdown, id()))->post();
84}
85
86status_t Converter::initCheck() const {
87    return mInitCheck;
88}
89
90size_t Converter::getInputBufferCount() const {
91    return mEncoderInputBuffers.size();
92}
93
94sp<AMessage> Converter::getOutputFormat() const {
95    return mOutputFormat;
96}
97
98bool Converter::needToManuallyPrependSPSPPS() const {
99    return mNeedToManuallyPrependSPSPPS;
100}
101
102static int32_t getBitrate(const char *propName, int32_t defaultValue) {
103    char val[PROPERTY_VALUE_MAX];
104    if (property_get(propName, val, NULL)) {
105        char *end;
106        unsigned long x = strtoul(val, &end, 10);
107
108        if (*end == '\0' && end > val && x > 0) {
109            return x;
110        }
111    }
112
113    return defaultValue;
114}
115
116status_t Converter::initEncoder() {
117    AString inputMIME;
118    CHECK(mInputFormat->findString("mime", &inputMIME));
119
120    AString outputMIME;
121    bool isAudio = false;
122    if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_AUDIO_RAW)) {
123        if (mIsPCMAudio) {
124            outputMIME = MEDIA_MIMETYPE_AUDIO_RAW;
125        } else {
126            outputMIME = MEDIA_MIMETYPE_AUDIO_AAC;
127        }
128        isAudio = true;
129    } else if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) {
130        outputMIME = MEDIA_MIMETYPE_VIDEO_AVC;
131    } else {
132        TRESPASS();
133    }
134
135    if (!mIsPCMAudio) {
136        mEncoder = MediaCodec::CreateByType(
137                mCodecLooper, outputMIME.c_str(), true /* encoder */);
138
139        if (mEncoder == NULL) {
140            return ERROR_UNSUPPORTED;
141        }
142    }
143
144    mOutputFormat = mInputFormat->dup();
145
146    if (mIsPCMAudio) {
147        return OK;
148    }
149
150    mOutputFormat->setString("mime", outputMIME.c_str());
151
152    int32_t audioBitrate = getBitrate("media.wfd.audio-bitrate", 128000);
153    int32_t videoBitrate = getBitrate("media.wfd.video-bitrate", 5000000);
154
155    ALOGI("using audio bitrate of %d bps, video bitrate of %d bps",
156          audioBitrate, videoBitrate);
157
158    if (isAudio) {
159        mOutputFormat->setInt32("bitrate", audioBitrate);
160    } else {
161        mOutputFormat->setInt32("bitrate", videoBitrate);
162        mOutputFormat->setInt32("bitrate-mode", OMX_Video_ControlRateConstant);
163        mOutputFormat->setInt32("frame-rate", 30);
164        mOutputFormat->setInt32("i-frame-interval", 15);  // Iframes every 15 secs
165
166        // Configure encoder to use intra macroblock refresh mode
167        mOutputFormat->setInt32("intra-refresh-mode", OMX_VIDEO_IntraRefreshCyclic);
168
169        int width, height, mbs;
170        if (!mOutputFormat->findInt32("width", &width)
171                || !mOutputFormat->findInt32("height", &height)) {
172            return ERROR_UNSUPPORTED;
173        }
174
175        // Update macroblocks in a cyclic fashion with 10% of all MBs within
176        // frame gets updated at one time. It takes about 10 frames to
177        // completely update a whole video frame. If the frame rate is 30,
178        // it takes about 333 ms in the best case (if next frame is not an IDR)
179        // to recover from a lost/corrupted packet.
180        mbs = (((width + 15) / 16) * ((height + 15) / 16) * 10) / 100;
181        mOutputFormat->setInt32("intra-refresh-CIR-mbs", mbs);
182    }
183
184    ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str());
185
186    mNeedToManuallyPrependSPSPPS = false;
187
188    status_t err = NO_INIT;
189
190    if (!isAudio) {
191        sp<AMessage> tmp = mOutputFormat->dup();
192        tmp->setInt32("prepend-sps-pps-to-idr-frames", 1);
193
194        err = mEncoder->configure(
195                tmp,
196                NULL /* nativeWindow */,
197                NULL /* crypto */,
198                MediaCodec::CONFIGURE_FLAG_ENCODE);
199
200        if (err == OK) {
201            // Encoder supported prepending SPS/PPS, we don't need to emulate
202            // it.
203            mOutputFormat = tmp;
204        } else {
205            mNeedToManuallyPrependSPSPPS = true;
206
207            ALOGI("We going to manually prepend SPS and PPS to IDR frames.");
208        }
209    }
210
211    if (err != OK) {
212        // We'll get here for audio or if we failed to configure the encoder
213        // to automatically prepend SPS/PPS in the case of video.
214
215        err = mEncoder->configure(
216                    mOutputFormat,
217                    NULL /* nativeWindow */,
218                    NULL /* crypto */,
219                    MediaCodec::CONFIGURE_FLAG_ENCODE);
220    }
221
222    if (err != OK) {
223        return err;
224    }
225
226    err = mEncoder->start();
227
228    if (err != OK) {
229        return err;
230    }
231
232    err = mEncoder->getInputBuffers(&mEncoderInputBuffers);
233
234    if (err != OK) {
235        return err;
236    }
237
238    return mEncoder->getOutputBuffers(&mEncoderOutputBuffers);
239}
240
241void Converter::notifyError(status_t err) {
242    sp<AMessage> notify = mNotify->dup();
243    notify->setInt32("what", kWhatError);
244    notify->setInt32("err", err);
245    notify->post();
246}
247
248// static
249bool Converter::IsSilence(const sp<ABuffer> &accessUnit) {
250    const uint8_t *ptr = accessUnit->data();
251    const uint8_t *end = ptr + accessUnit->size();
252    while (ptr < end) {
253        if (*ptr != 0) {
254            return false;
255        }
256        ++ptr;
257    }
258
259    return true;
260}
261
262void Converter::onMessageReceived(const sp<AMessage> &msg) {
263    switch (msg->what()) {
264        case kWhatMediaPullerNotify:
265        {
266            int32_t what;
267            CHECK(msg->findInt32("what", &what));
268
269            if (!mIsPCMAudio && mEncoder == NULL) {
270                ALOGV("got msg '%s' after encoder shutdown.",
271                      msg->debugString().c_str());
272
273                if (what == MediaPuller::kWhatAccessUnit) {
274                    sp<ABuffer> accessUnit;
275                    CHECK(msg->findBuffer("accessUnit", &accessUnit));
276
277                    void *mbuf;
278                    if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf)
279                            && mbuf != NULL) {
280                        ALOGV("releasing mbuf %p", mbuf);
281
282                        accessUnit->meta()->setPointer("mediaBuffer", NULL);
283
284                        static_cast<MediaBuffer *>(mbuf)->release();
285                        mbuf = NULL;
286                    }
287                }
288                break;
289            }
290
291            if (what == MediaPuller::kWhatEOS) {
292                mInputBufferQueue.push_back(NULL);
293
294                feedEncoderInputBuffers();
295
296                scheduleDoMoreWork();
297            } else {
298                CHECK_EQ(what, MediaPuller::kWhatAccessUnit);
299
300                sp<ABuffer> accessUnit;
301                CHECK(msg->findBuffer("accessUnit", &accessUnit));
302
303#if 0
304                void *mbuf;
305                if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf)
306                        && mbuf != NULL) {
307                    ALOGI("queueing mbuf %p", mbuf);
308                }
309#endif
310
311#if ENABLE_SILENCE_DETECTION
312                if (!mIsVideo) {
313                    if (IsSilence(accessUnit)) {
314                        if (mInSilentMode) {
315                            break;
316                        }
317
318                        int64_t nowUs = ALooper::GetNowUs();
319
320                        if (mFirstSilentFrameUs < 0ll) {
321                            mFirstSilentFrameUs = nowUs;
322                        } else if (nowUs >= mFirstSilentFrameUs + 10000000ll) {
323                            mInSilentMode = true;
324                            ALOGI("audio in silent mode now.");
325                            break;
326                        }
327                    } else {
328                        if (mInSilentMode) {
329                            ALOGI("audio no longer in silent mode.");
330                        }
331                        mInSilentMode = false;
332                        mFirstSilentFrameUs = -1ll;
333                    }
334                }
335#endif
336
337                mInputBufferQueue.push_back(accessUnit);
338
339                feedEncoderInputBuffers();
340
341                scheduleDoMoreWork();
342            }
343            break;
344        }
345
346        case kWhatEncoderActivity:
347        {
348#if 0
349            int64_t whenUs;
350            if (msg->findInt64("whenUs", &whenUs)) {
351                int64_t nowUs = ALooper::GetNowUs();
352                ALOGI("[%s] kWhatEncoderActivity after %lld us",
353                      mIsVideo ? "video" : "audio", nowUs - whenUs);
354            }
355#endif
356
357            mDoMoreWorkPending = false;
358
359            if (mEncoder == NULL) {
360                break;
361            }
362
363            status_t err = doMoreWork();
364
365            if (err != OK) {
366                notifyError(err);
367            } else {
368                scheduleDoMoreWork();
369            }
370            break;
371        }
372
373        case kWhatRequestIDRFrame:
374        {
375            if (mEncoder == NULL) {
376                break;
377            }
378
379            if (mIsVideo) {
380                ALOGI("requesting IDR frame");
381                mEncoder->requestIDRFrame();
382            }
383            break;
384        }
385
386        case kWhatShutdown:
387        {
388            ALOGI("shutting down encoder");
389
390            if (mEncoder != NULL) {
391                mEncoder->release();
392                mEncoder.clear();
393            }
394
395            AString mime;
396            CHECK(mInputFormat->findString("mime", &mime));
397            ALOGI("encoder (%s) shut down.", mime.c_str());
398            break;
399        }
400
401        default:
402            TRESPASS();
403    }
404}
405
406void Converter::scheduleDoMoreWork() {
407    if (mIsPCMAudio) {
408        // There's no encoder involved in this case.
409        return;
410    }
411
412    if (mDoMoreWorkPending) {
413        return;
414    }
415
416    mDoMoreWorkPending = true;
417
418#if 1
419    if (mEncoderActivityNotify == NULL) {
420        mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, id());
421    }
422    mEncoder->requestActivityNotification(mEncoderActivityNotify->dup());
423#else
424    sp<AMessage> notify = new AMessage(kWhatEncoderActivity, id());
425    notify->setInt64("whenUs", ALooper::GetNowUs());
426    mEncoder->requestActivityNotification(notify);
427#endif
428}
429
430status_t Converter::feedRawAudioInputBuffers() {
431    // Split incoming PCM audio into buffers of 6 AUs of 80 audio frames each
432    // and add a 4 byte header according to the wifi display specs.
433
434    while (!mInputBufferQueue.empty()) {
435        sp<ABuffer> buffer = *mInputBufferQueue.begin();
436        mInputBufferQueue.erase(mInputBufferQueue.begin());
437
438        int16_t *ptr = (int16_t *)buffer->data();
439        int16_t *stop = (int16_t *)(buffer->data() + buffer->size());
440        while (ptr < stop) {
441            *ptr = htons(*ptr);
442            ++ptr;
443        }
444
445        static const size_t kFrameSize = 2 * sizeof(int16_t);  // stereo
446        static const size_t kFramesPerAU = 80;
447        static const size_t kNumAUsPerPESPacket = 6;
448
449        if (mPartialAudioAU != NULL) {
450            size_t bytesMissingForFullAU =
451                kNumAUsPerPESPacket * kFramesPerAU * kFrameSize
452                - mPartialAudioAU->size() + 4;
453
454            size_t copy = buffer->size();
455            if(copy > bytesMissingForFullAU) {
456                copy = bytesMissingForFullAU;
457            }
458
459            memcpy(mPartialAudioAU->data() + mPartialAudioAU->size(),
460                   buffer->data(),
461                   copy);
462
463            mPartialAudioAU->setRange(0, mPartialAudioAU->size() + copy);
464
465            buffer->setRange(buffer->offset() + copy, buffer->size() - copy);
466
467            int64_t timeUs;
468            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
469
470            int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0);
471            timeUs += copyUs;
472            buffer->meta()->setInt64("timeUs", timeUs);
473
474            if (bytesMissingForFullAU == copy) {
475                sp<AMessage> notify = mNotify->dup();
476                notify->setInt32("what", kWhatAccessUnit);
477                notify->setBuffer("accessUnit", mPartialAudioAU);
478                notify->post();
479
480                mPartialAudioAU.clear();
481            }
482        }
483
484        while (buffer->size() > 0) {
485            sp<ABuffer> partialAudioAU =
486                new ABuffer(
487                        4
488                        + kNumAUsPerPESPacket * kFrameSize * kFramesPerAU);
489
490            uint8_t *ptr = partialAudioAU->data();
491            ptr[0] = 0xa0;  // 10100000b
492            ptr[1] = kNumAUsPerPESPacket;
493            ptr[2] = 0;  // reserved, audio _emphasis_flag = 0
494
495            static const unsigned kQuantizationWordLength = 0;  // 16-bit
496            static const unsigned kAudioSamplingFrequency = 2;  // 48Khz
497            static const unsigned kNumberOfAudioChannels = 1;  // stereo
498
499            ptr[3] = (kQuantizationWordLength << 6)
500                    | (kAudioSamplingFrequency << 3)
501                    | kNumberOfAudioChannels;
502
503            size_t copy = buffer->size();
504            if (copy > partialAudioAU->size() - 4) {
505                copy = partialAudioAU->size() - 4;
506            }
507
508            memcpy(&ptr[4], buffer->data(), copy);
509
510            partialAudioAU->setRange(0, 4 + copy);
511            buffer->setRange(buffer->offset() + copy, buffer->size() - copy);
512
513            int64_t timeUs;
514            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
515
516            partialAudioAU->meta()->setInt64("timeUs", timeUs);
517
518            int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0);
519            timeUs += copyUs;
520            buffer->meta()->setInt64("timeUs", timeUs);
521
522            if (copy == partialAudioAU->capacity() - 4) {
523                sp<AMessage> notify = mNotify->dup();
524                notify->setInt32("what", kWhatAccessUnit);
525                notify->setBuffer("accessUnit", partialAudioAU);
526                notify->post();
527
528                partialAudioAU.clear();
529                continue;
530            }
531
532            mPartialAudioAU = partialAudioAU;
533        }
534    }
535
536    return OK;
537}
538
539status_t Converter::feedEncoderInputBuffers() {
540    if (mIsPCMAudio) {
541        return feedRawAudioInputBuffers();
542    }
543
544    while (!mInputBufferQueue.empty()
545            && !mAvailEncoderInputIndices.empty()) {
546        sp<ABuffer> buffer = *mInputBufferQueue.begin();
547        mInputBufferQueue.erase(mInputBufferQueue.begin());
548
549        size_t bufferIndex = *mAvailEncoderInputIndices.begin();
550        mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin());
551
552        int64_t timeUs = 0ll;
553        uint32_t flags = 0;
554
555        if (buffer != NULL) {
556            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
557
558            memcpy(mEncoderInputBuffers.itemAt(bufferIndex)->data(),
559                   buffer->data(),
560                   buffer->size());
561
562            void *mediaBuffer;
563            if (buffer->meta()->findPointer("mediaBuffer", &mediaBuffer)
564                    && mediaBuffer != NULL) {
565                mEncoderInputBuffers.itemAt(bufferIndex)->meta()
566                    ->setPointer("mediaBuffer", mediaBuffer);
567
568                buffer->meta()->setPointer("mediaBuffer", NULL);
569            }
570        } else {
571            flags = MediaCodec::BUFFER_FLAG_EOS;
572        }
573
574        status_t err = mEncoder->queueInputBuffer(
575                bufferIndex, 0, (buffer == NULL) ? 0 : buffer->size(),
576                timeUs, flags);
577
578        if (err != OK) {
579            return err;
580        }
581    }
582
583    return OK;
584}
585
586status_t Converter::doMoreWork() {
587    status_t err;
588
589    for (;;) {
590        size_t bufferIndex;
591        err = mEncoder->dequeueInputBuffer(&bufferIndex);
592
593        if (err != OK) {
594            break;
595        }
596
597        mAvailEncoderInputIndices.push_back(bufferIndex);
598    }
599
600    feedEncoderInputBuffers();
601
602    for (;;) {
603        size_t bufferIndex;
604        size_t offset;
605        size_t size;
606        int64_t timeUs;
607        uint32_t flags;
608        err = mEncoder->dequeueOutputBuffer(
609                &bufferIndex, &offset, &size, &timeUs, &flags);
610
611        if (err != OK) {
612            if (err == -EAGAIN) {
613                err = OK;
614            }
615            break;
616        }
617
618        if (flags & MediaCodec::BUFFER_FLAG_EOS) {
619            sp<AMessage> notify = mNotify->dup();
620            notify->setInt32("what", kWhatEOS);
621            notify->post();
622        } else {
623            sp<ABuffer> buffer = new ABuffer(size);
624            buffer->meta()->setInt64("timeUs", timeUs);
625
626            ALOGV("[%s] time %lld us (%.2f secs)",
627                  mIsVideo ? "video" : "audio", timeUs, timeUs / 1E6);
628
629            memcpy(buffer->data(),
630                   mEncoderOutputBuffers.itemAt(bufferIndex)->base() + offset,
631                   size);
632
633            if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) {
634                mOutputFormat->setBuffer("csd-0", buffer);
635            } else {
636                sp<AMessage> notify = mNotify->dup();
637                notify->setInt32("what", kWhatAccessUnit);
638                notify->setBuffer("accessUnit", buffer);
639                notify->post();
640            }
641        }
642
643        mEncoder->releaseOutputBuffer(bufferIndex);
644
645        if (flags & MediaCodec::BUFFER_FLAG_EOS) {
646            break;
647        }
648    }
649
650    return err;
651}
652
653void Converter::requestIDRFrame() {
654    (new AMessage(kWhatRequestIDRFrame, id()))->post();
655}
656
657}  // namespace android
658