Converter.cpp revision 90a92053219ae50ddf4bb54e3d54db2d309e2b8d 
1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "Converter"
19#include <utils/Log.h>
20
21#include "Converter.h"
22
23#include "MediaPuller.h"
24
25#include <cutils/properties.h>
26#include <gui/SurfaceTextureClient.h>
27#include <media/ICrypto.h>
28#include <media/stagefright/foundation/ABuffer.h>
29#include <media/stagefright/foundation/ADebug.h>
30#include <media/stagefright/foundation/AMessage.h>
31#include <media/stagefright/MediaBuffer.h>
32#include <media/stagefright/MediaCodec.h>
33#include <media/stagefright/MediaDefs.h>
34#include <media/stagefright/MediaErrors.h>
35
36namespace android {
37
38Converter::Converter(
39        const sp<AMessage> &notify,
40        const sp<ALooper> &codecLooper,
41        const sp<AMessage> &format,
42        bool usePCMAudio)
43    : mInitCheck(NO_INIT),
44      mNotify(notify),
45      mCodecLooper(codecLooper),
46      mInputFormat(format),
47      mIsVideo(false),
48      mIsPCMAudio(usePCMAudio),
49      mDoMoreWorkPending(false)
50#if ENABLE_SILENCE_DETECTION
51      ,mFirstSilentFrameUs(-1ll)
52      ,mInSilentMode(false)
53#endif
54    {
55    AString mime;
56    CHECK(mInputFormat->findString("mime", &mime));
57
58    if (!strncasecmp("video/", mime.c_str(), 6)) {
59        mIsVideo = true;
60    }
61
62    CHECK(!usePCMAudio || !mIsVideo);
63
64    mInitCheck = initEncoder();
65
66    if (mInitCheck != OK) {
67        if (mEncoder != NULL) {
68            mEncoder->release();
69            mEncoder.clear();
70        }
71    }
72}
73
74Converter::~Converter() {
75    CHECK(mEncoder == NULL);
76}
77
78void Converter::shutdownAsync() {
79    ALOGV("shutdown");
80    (new AMessage(kWhatShutdown, id()))->post();
81}
82
83status_t Converter::initCheck() const {
84    return mInitCheck;
85}
86
87size_t Converter::getInputBufferCount() const {
88    return mEncoderInputBuffers.size();
89}
90
91sp<AMessage> Converter::getOutputFormat() const {
92    return mOutputFormat;
93}
94
95static int32_t getBitrate(const char *propName, int32_t defaultValue) {
96    char val[PROPERTY_VALUE_MAX];
97    if (property_get(propName, val, NULL)) {
98        char *end;
99        unsigned long x = strtoul(val, &end, 10);
100
101        if (*end == '\0' && end > val && x > 0) {
102            return x;
103        }
104    }
105
106    return defaultValue;
107}
108
109status_t Converter::initEncoder() {
110    AString inputMIME;
111    CHECK(mInputFormat->findString("mime", &inputMIME));
112
113    AString outputMIME;
114    bool isAudio = false;
115    if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_AUDIO_RAW)) {
116        if (mIsPCMAudio) {
117            outputMIME = MEDIA_MIMETYPE_AUDIO_RAW;
118        } else {
119            outputMIME = MEDIA_MIMETYPE_AUDIO_AAC;
120        }
121        isAudio = true;
122    } else if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) {
123        outputMIME = MEDIA_MIMETYPE_VIDEO_AVC;
124    } else {
125        TRESPASS();
126    }
127
128    if (!mIsPCMAudio) {
129        mEncoder = MediaCodec::CreateByType(
130                mCodecLooper, outputMIME.c_str(), true /* encoder */);
131
132        if (mEncoder == NULL) {
133            return ERROR_UNSUPPORTED;
134        }
135    }
136
137    mOutputFormat = mInputFormat->dup();
138
139    if (mIsPCMAudio) {
140        return OK;
141    }
142
143    mOutputFormat->setString("mime", outputMIME.c_str());
144
145    int32_t audioBitrate = getBitrate("media.wfd.audio-bitrate", 128000);
146    int32_t videoBitrate = getBitrate("media.wfd.video-bitrate", 5000000);
147
148    ALOGI("using audio bitrate of %d bps, video bitrate of %d bps",
149          audioBitrate, videoBitrate);
150
151    if (isAudio) {
152        mOutputFormat->setInt32("bitrate", audioBitrate);
153    } else {
154        mOutputFormat->setInt32("bitrate", videoBitrate);
155        mOutputFormat->setInt32("frame-rate", 30);
156        mOutputFormat->setInt32("i-frame-interval", 1);  // Iframes every 1 secs
157        mOutputFormat->setInt32("prepend-sps-pps-to-idr-frames", 1);
158    }
159
160    ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str());
161
162    status_t err = mEncoder->configure(
163            mOutputFormat,
164            NULL /* nativeWindow */,
165            NULL /* crypto */,
166            MediaCodec::CONFIGURE_FLAG_ENCODE);
167
168    if (err != OK) {
169        return err;
170    }
171
172    err = mEncoder->start();
173
174    if (err != OK) {
175        return err;
176    }
177
178    err = mEncoder->getInputBuffers(&mEncoderInputBuffers);
179
180    if (err != OK) {
181        return err;
182    }
183
184    return mEncoder->getOutputBuffers(&mEncoderOutputBuffers);
185}
186
187void Converter::notifyError(status_t err) {
188    sp<AMessage> notify = mNotify->dup();
189    notify->setInt32("what", kWhatError);
190    notify->setInt32("err", err);
191    notify->post();
192}
193
194// static
195bool Converter::IsSilence(const sp<ABuffer> &accessUnit) {
196    const uint8_t *ptr = accessUnit->data();
197    const uint8_t *end = ptr + accessUnit->size();
198    while (ptr < end) {
199        if (*ptr != 0) {
200            return false;
201        }
202        ++ptr;
203    }
204
205    return true;
206}
207
208void Converter::onMessageReceived(const sp<AMessage> &msg) {
209    switch (msg->what()) {
210        case kWhatMediaPullerNotify:
211        {
212            int32_t what;
213            CHECK(msg->findInt32("what", &what));
214
215            if (!mIsPCMAudio && mEncoder == NULL) {
216                ALOGV("got msg '%s' after encoder shutdown.",
217                      msg->debugString().c_str());
218
219                if (what == MediaPuller::kWhatAccessUnit) {
220                    sp<ABuffer> accessUnit;
221                    CHECK(msg->findBuffer("accessUnit", &accessUnit));
222
223                    void *mbuf;
224                    if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf)
225                            && mbuf != NULL) {
226                        ALOGV("releasing mbuf %p", mbuf);
227
228                        accessUnit->meta()->setPointer("mediaBuffer", NULL);
229
230                        static_cast<MediaBuffer *>(mbuf)->release();
231                        mbuf = NULL;
232                    }
233                }
234                break;
235            }
236
237            if (what == MediaPuller::kWhatEOS) {
238                mInputBufferQueue.push_back(NULL);
239
240                feedEncoderInputBuffers();
241
242                scheduleDoMoreWork();
243            } else {
244                CHECK_EQ(what, MediaPuller::kWhatAccessUnit);
245
246                sp<ABuffer> accessUnit;
247                CHECK(msg->findBuffer("accessUnit", &accessUnit));
248
249#if 0
250                void *mbuf;
251                if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf)
252                        && mbuf != NULL) {
253                    ALOGI("queueing mbuf %p", mbuf);
254                }
255#endif
256
257#if ENABLE_SILENCE_DETECTION
258                if (!mIsVideo) {
259                    if (IsSilence(accessUnit)) {
260                        if (mInSilentMode) {
261                            break;
262                        }
263
264                        int64_t nowUs = ALooper::GetNowUs();
265
266                        if (mFirstSilentFrameUs < 0ll) {
267                            mFirstSilentFrameUs = nowUs;
268                        } else if (nowUs >= mFirstSilentFrameUs + 10000000ll) {
269                            mInSilentMode = true;
270                            ALOGI("audio in silent mode now.");
271                            break;
272                        }
273                    } else {
274                        if (mInSilentMode) {
275                            ALOGI("audio no longer in silent mode.");
276                        }
277                        mInSilentMode = false;
278                        mFirstSilentFrameUs = -1ll;
279                    }
280                }
281#endif
282
283                mInputBufferQueue.push_back(accessUnit);
284
285                feedEncoderInputBuffers();
286
287                scheduleDoMoreWork();
288            }
289            break;
290        }
291
292        case kWhatEncoderActivity:
293        {
294#if 0
295            int64_t whenUs;
296            if (msg->findInt64("whenUs", &whenUs)) {
297                int64_t nowUs = ALooper::GetNowUs();
298                ALOGI("[%s] kWhatEncoderActivity after %lld us",
299                      mIsVideo ? "video" : "audio", nowUs - whenUs);
300            }
301#endif
302
303            mDoMoreWorkPending = false;
304
305            if (mEncoder == NULL) {
306                break;
307            }
308
309            status_t err = doMoreWork();
310
311            if (err != OK) {
312                notifyError(err);
313            } else {
314                scheduleDoMoreWork();
315            }
316            break;
317        }
318
319        case kWhatRequestIDRFrame:
320        {
321            if (mEncoder == NULL) {
322                break;
323            }
324
325            if (mIsVideo) {
326                ALOGI("requesting IDR frame");
327                mEncoder->requestIDRFrame();
328            }
329            break;
330        }
331
332        case kWhatShutdown:
333        {
334            ALOGI("shutting down encoder");
335
336            if (mEncoder != NULL) {
337                mEncoder->release();
338                mEncoder.clear();
339            }
340
341            AString mime;
342            CHECK(mInputFormat->findString("mime", &mime));
343            ALOGI("encoder (%s) shut down.", mime.c_str());
344            break;
345        }
346
347        default:
348            TRESPASS();
349    }
350}
351
352void Converter::scheduleDoMoreWork() {
353    if (mIsPCMAudio) {
354        // There's no encoder involved in this case.
355        return;
356    }
357
358    if (mDoMoreWorkPending) {
359        return;
360    }
361
362    mDoMoreWorkPending = true;
363
364#if 1
365    if (mEncoderActivityNotify == NULL) {
366        mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, id());
367    }
368    mEncoder->requestActivityNotification(mEncoderActivityNotify->dup());
369#else
370    sp<AMessage> notify = new AMessage(kWhatEncoderActivity, id());
371    notify->setInt64("whenUs", ALooper::GetNowUs());
372    mEncoder->requestActivityNotification(notify);
373#endif
374}
375
376status_t Converter::feedRawAudioInputBuffers() {
377    // Split incoming PCM audio into buffers of 6 AUs of 80 audio frames each
378    // and add a 4 byte header according to the wifi display specs.
379
380    while (!mInputBufferQueue.empty()) {
381        sp<ABuffer> buffer = *mInputBufferQueue.begin();
382        mInputBufferQueue.erase(mInputBufferQueue.begin());
383
384        int16_t *ptr = (int16_t *)buffer->data();
385        int16_t *stop = (int16_t *)(buffer->data() + buffer->size());
386        while (ptr < stop) {
387            *ptr = htons(*ptr);
388            ++ptr;
389        }
390
391        static const size_t kFrameSize = 2 * sizeof(int16_t);  // stereo
392        static const size_t kFramesPerAU = 80;
393        static const size_t kNumAUsPerPESPacket = 6;
394
395        if (mPartialAudioAU != NULL) {
396            size_t bytesMissingForFullAU =
397                kNumAUsPerPESPacket * kFramesPerAU * kFrameSize
398                - mPartialAudioAU->size() + 4;
399
400            size_t copy = buffer->size();
401            if(copy > bytesMissingForFullAU) {
402                copy = bytesMissingForFullAU;
403            }
404
405            memcpy(mPartialAudioAU->data() + mPartialAudioAU->size(),
406                   buffer->data(),
407                   copy);
408
409            mPartialAudioAU->setRange(0, mPartialAudioAU->size() + copy);
410
411            buffer->setRange(buffer->offset() + copy, buffer->size() - copy);
412
413            int64_t timeUs;
414            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
415
416            int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0);
417            timeUs += copyUs;
418            buffer->meta()->setInt64("timeUs", timeUs);
419
420            if (bytesMissingForFullAU == copy) {
421                sp<AMessage> notify = mNotify->dup();
422                notify->setInt32("what", kWhatAccessUnit);
423                notify->setBuffer("accessUnit", mPartialAudioAU);
424                notify->post();
425
426                mPartialAudioAU.clear();
427            }
428        }
429
430        while (buffer->size() > 0) {
431            sp<ABuffer> partialAudioAU =
432                new ABuffer(
433                        4
434                        + kNumAUsPerPESPacket * kFrameSize * kFramesPerAU);
435
436            uint8_t *ptr = partialAudioAU->data();
437            ptr[0] = 0xa0;  // 10100000b
438            ptr[1] = kNumAUsPerPESPacket;
439            ptr[2] = 0;  // reserved, audio _emphasis_flag = 0
440
441            static const unsigned kQuantizationWordLength = 0;  // 16-bit
442            static const unsigned kAudioSamplingFrequency = 2;  // 48Khz
443            static const unsigned kNumberOfAudioChannels = 1;  // stereo
444
445            ptr[3] = (kQuantizationWordLength << 6)
446                    | (kAudioSamplingFrequency << 3)
447                    | kNumberOfAudioChannels;
448
449            size_t copy = buffer->size();
450            if (copy > partialAudioAU->size() - 4) {
451                copy = partialAudioAU->size() - 4;
452            }
453
454            memcpy(&ptr[4], buffer->data(), copy);
455
456            partialAudioAU->setRange(0, 4 + copy);
457            buffer->setRange(buffer->offset() + copy, buffer->size() - copy);
458
459            int64_t timeUs;
460            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
461
462            partialAudioAU->meta()->setInt64("timeUs", timeUs);
463
464            int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0);
465            timeUs += copyUs;
466            buffer->meta()->setInt64("timeUs", timeUs);
467
468            if (copy == partialAudioAU->capacity() - 4) {
469                sp<AMessage> notify = mNotify->dup();
470                notify->setInt32("what", kWhatAccessUnit);
471                notify->setBuffer("accessUnit", partialAudioAU);
472                notify->post();
473
474                partialAudioAU.clear();
475                continue;
476            }
477
478            mPartialAudioAU = partialAudioAU;
479        }
480    }
481
482    return OK;
483}
484
485status_t Converter::feedEncoderInputBuffers() {
486    if (mIsPCMAudio) {
487        return feedRawAudioInputBuffers();
488    }
489
490    while (!mInputBufferQueue.empty()
491            && !mAvailEncoderInputIndices.empty()) {
492        sp<ABuffer> buffer = *mInputBufferQueue.begin();
493        mInputBufferQueue.erase(mInputBufferQueue.begin());
494
495        size_t bufferIndex = *mAvailEncoderInputIndices.begin();
496        mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin());
497
498        int64_t timeUs = 0ll;
499        uint32_t flags = 0;
500
501        if (buffer != NULL) {
502            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
503
504            memcpy(mEncoderInputBuffers.itemAt(bufferIndex)->data(),
505                   buffer->data(),
506                   buffer->size());
507
508            void *mediaBuffer;
509            if (buffer->meta()->findPointer("mediaBuffer", &mediaBuffer)
510                    && mediaBuffer != NULL) {
511                mEncoderInputBuffers.itemAt(bufferIndex)->meta()
512                    ->setPointer("mediaBuffer", mediaBuffer);
513
514                buffer->meta()->setPointer("mediaBuffer", NULL);
515            }
516        } else {
517            flags = MediaCodec::BUFFER_FLAG_EOS;
518        }
519
520        status_t err = mEncoder->queueInputBuffer(
521                bufferIndex, 0, (buffer == NULL) ? 0 : buffer->size(),
522                timeUs, flags);
523
524        if (err != OK) {
525            return err;
526        }
527    }
528
529    return OK;
530}
531
532status_t Converter::doMoreWork() {
533    status_t err;
534
535    for (;;) {
536        size_t bufferIndex;
537        err = mEncoder->dequeueInputBuffer(&bufferIndex);
538
539        if (err != OK) {
540            break;
541        }
542
543        mAvailEncoderInputIndices.push_back(bufferIndex);
544    }
545
546    feedEncoderInputBuffers();
547
548    for (;;) {
549        size_t bufferIndex;
550        size_t offset;
551        size_t size;
552        int64_t timeUs;
553        uint32_t flags;
554        err = mEncoder->dequeueOutputBuffer(
555                &bufferIndex, &offset, &size, &timeUs, &flags);
556
557        if (err != OK) {
558            if (err == -EAGAIN) {
559                err = OK;
560            }
561            break;
562        }
563
564        if (flags & MediaCodec::BUFFER_FLAG_EOS) {
565            sp<AMessage> notify = mNotify->dup();
566            notify->setInt32("what", kWhatEOS);
567            notify->post();
568        } else {
569            sp<ABuffer> buffer = new ABuffer(size);
570            buffer->meta()->setInt64("timeUs", timeUs);
571
572            ALOGV("[%s] time %lld us (%.2f secs)",
573                  mIsVideo ? "video" : "audio", timeUs, timeUs / 1E6);
574
575            memcpy(buffer->data(),
576                   mEncoderOutputBuffers.itemAt(bufferIndex)->base() + offset,
577                   size);
578
579            if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) {
580                mOutputFormat->setBuffer("csd-0", buffer);
581            } else {
582                sp<AMessage> notify = mNotify->dup();
583                notify->setInt32("what", kWhatAccessUnit);
584                notify->setBuffer("accessUnit", buffer);
585                notify->post();
586            }
587        }
588
589        mEncoder->releaseOutputBuffer(bufferIndex);
590
591        if (flags & MediaCodec::BUFFER_FLAG_EOS) {
592            break;
593        }
594    }
595
596    return err;
597}
598
599void Converter::requestIDRFrame() {
600    (new AMessage(kWhatRequestIDRFrame, id()))->post();
601}
602
603}  // namespace android
604