1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "SimplePlayer"
19#include <utils/Log.h>
20
21#include "SimplePlayer.h"
22
23#include <gui/Surface.h>
24
25#include <media/AudioTrack.h>
26#include <media/ICrypto.h>
27#include <media/IMediaHTTPService.h>
28#include <media/MediaCodecBuffer.h>
29#include <media/stagefright/foundation/ABuffer.h>
30#include <media/stagefright/foundation/ADebug.h>
31#include <media/stagefright/foundation/AMessage.h>
32#include <media/stagefright/MediaCodec.h>
33#include <media/stagefright/MediaErrors.h>
34#include <media/stagefright/NuMediaExtractor.h>
35
36namespace android {
37
38SimplePlayer::SimplePlayer()
39    : mState(UNINITIALIZED),
40      mDoMoreStuffGeneration(0),
41      mStartTimeRealUs(-1ll) {
42}
43
44SimplePlayer::~SimplePlayer() {
45}
46
47// static
48status_t PostAndAwaitResponse(
49        const sp<AMessage> &msg, sp<AMessage> *response) {
50    status_t err = msg->postAndAwaitResponse(response);
51
52    if (err != OK) {
53        return err;
54    }
55
56    if (!(*response)->findInt32("err", &err)) {
57        err = OK;
58    }
59
60    return err;
61}
62status_t SimplePlayer::setDataSource(const char *path) {
63    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
64    msg->setString("path", path);
65    sp<AMessage> response;
66    return PostAndAwaitResponse(msg, &response);
67}
68
69status_t SimplePlayer::setSurface(const sp<IGraphicBufferProducer> &bufferProducer) {
70    sp<AMessage> msg = new AMessage(kWhatSetSurface, this);
71
72    sp<Surface> surface;
73    if (bufferProducer != NULL) {
74        surface = new Surface(bufferProducer);
75    }
76
77    msg->setObject("surface", surface);
78
79    sp<AMessage> response;
80    return PostAndAwaitResponse(msg, &response);
81}
82
83status_t SimplePlayer::prepare() {
84    sp<AMessage> msg = new AMessage(kWhatPrepare, this);
85    sp<AMessage> response;
86    return PostAndAwaitResponse(msg, &response);
87}
88
89status_t SimplePlayer::start() {
90    sp<AMessage> msg = new AMessage(kWhatStart, this);
91    sp<AMessage> response;
92    return PostAndAwaitResponse(msg, &response);
93}
94
95status_t SimplePlayer::stop() {
96    sp<AMessage> msg = new AMessage(kWhatStop, this);
97    sp<AMessage> response;
98    return PostAndAwaitResponse(msg, &response);
99}
100
101status_t SimplePlayer::reset() {
102    sp<AMessage> msg = new AMessage(kWhatReset, this);
103    sp<AMessage> response;
104    return PostAndAwaitResponse(msg, &response);
105}
106
107void SimplePlayer::onMessageReceived(const sp<AMessage> &msg) {
108    switch (msg->what()) {
109        case kWhatSetDataSource:
110        {
111            status_t err;
112            if (mState != UNINITIALIZED) {
113                err = INVALID_OPERATION;
114            } else {
115                CHECK(msg->findString("path", &mPath));
116                mState = UNPREPARED;
117            }
118
119            sp<AReplyToken> replyID;
120            CHECK(msg->senderAwaitsResponse(&replyID));
121
122            sp<AMessage> response = new AMessage;
123            response->setInt32("err", err);
124            response->postReply(replyID);
125            break;
126        }
127
128        case kWhatSetSurface:
129        {
130            status_t err;
131            if (mState != UNPREPARED) {
132                err = INVALID_OPERATION;
133            } else {
134                sp<RefBase> obj;
135                CHECK(msg->findObject("surface", &obj));
136                mSurface = static_cast<Surface *>(obj.get());
137                err = OK;
138            }
139
140            sp<AReplyToken> replyID;
141            CHECK(msg->senderAwaitsResponse(&replyID));
142
143            sp<AMessage> response = new AMessage;
144            response->setInt32("err", err);
145            response->postReply(replyID);
146            break;
147        }
148
149        case kWhatPrepare:
150        {
151            status_t err;
152            if (mState != UNPREPARED) {
153                err = INVALID_OPERATION;
154            } else {
155                err = onPrepare();
156
157                if (err == OK) {
158                    mState = STOPPED;
159                }
160            }
161
162            sp<AReplyToken> replyID;
163            CHECK(msg->senderAwaitsResponse(&replyID));
164
165            sp<AMessage> response = new AMessage;
166            response->setInt32("err", err);
167            response->postReply(replyID);
168            break;
169        }
170
171        case kWhatStart:
172        {
173            status_t err = OK;
174
175            if (mState == UNPREPARED) {
176                err = onPrepare();
177
178                if (err == OK) {
179                    mState = STOPPED;
180                }
181            }
182
183            if (err == OK) {
184                if (mState != STOPPED) {
185                    err = INVALID_OPERATION;
186                } else {
187                    err = onStart();
188
189                    if (err == OK) {
190                        mState = STARTED;
191                    }
192                }
193            }
194
195            sp<AReplyToken> replyID;
196            CHECK(msg->senderAwaitsResponse(&replyID));
197
198            sp<AMessage> response = new AMessage;
199            response->setInt32("err", err);
200            response->postReply(replyID);
201            break;
202        }
203
204        case kWhatStop:
205        {
206            status_t err;
207
208            if (mState != STARTED) {
209                err = INVALID_OPERATION;
210            } else {
211                err = onStop();
212
213                if (err == OK) {
214                    mState = STOPPED;
215                }
216            }
217
218            sp<AReplyToken> replyID;
219            CHECK(msg->senderAwaitsResponse(&replyID));
220
221            sp<AMessage> response = new AMessage;
222            response->setInt32("err", err);
223            response->postReply(replyID);
224            break;
225        }
226
227        case kWhatReset:
228        {
229            status_t err = OK;
230
231            if (mState == STARTED) {
232                CHECK_EQ(onStop(), (status_t)OK);
233                mState = STOPPED;
234            }
235
236            if (mState == STOPPED) {
237                err = onReset();
238                mState = UNINITIALIZED;
239            }
240
241            sp<AReplyToken> replyID;
242            CHECK(msg->senderAwaitsResponse(&replyID));
243
244            sp<AMessage> response = new AMessage;
245            response->setInt32("err", err);
246            response->postReply(replyID);
247            break;
248        }
249
250        case kWhatDoMoreStuff:
251        {
252            int32_t generation;
253            CHECK(msg->findInt32("generation", &generation));
254
255            if (generation != mDoMoreStuffGeneration) {
256                break;
257            }
258
259            status_t err = onDoMoreStuff();
260
261            if (err == OK) {
262                msg->post(10000ll);
263            }
264            break;
265        }
266
267        default:
268            TRESPASS();
269    }
270}
271
272status_t SimplePlayer::onPrepare() {
273    CHECK_EQ(mState, UNPREPARED);
274
275    mExtractor = new NuMediaExtractor;
276
277    status_t err = mExtractor->setDataSource(
278            NULL /* httpService */, mPath.c_str());
279
280    if (err != OK) {
281        mExtractor.clear();
282        return err;
283    }
284
285    if (mCodecLooper == NULL) {
286        mCodecLooper = new ALooper;
287        mCodecLooper->start();
288    }
289
290    bool haveAudio = false;
291    bool haveVideo = false;
292    for (size_t i = 0; i < mExtractor->countTracks(); ++i) {
293        sp<AMessage> format;
294        status_t err = mExtractor->getTrackFormat(i, &format);
295        CHECK_EQ(err, (status_t)OK);
296
297        AString mime;
298        CHECK(format->findString("mime", &mime));
299
300        bool isVideo = !strncasecmp(mime.c_str(), "video/", 6);
301
302        if (!haveAudio && !strncasecmp(mime.c_str(), "audio/", 6)) {
303            haveAudio = true;
304        } else if (!haveVideo && isVideo) {
305            haveVideo = true;
306        } else {
307            continue;
308        }
309
310        err = mExtractor->selectTrack(i);
311        CHECK_EQ(err, (status_t)OK);
312
313        CodecState *state =
314            &mStateByTrackIndex.editValueAt(
315                    mStateByTrackIndex.add(i, CodecState()));
316
317        state->mNumFramesWritten = 0;
318        state->mCodec = MediaCodec::CreateByType(
319                mCodecLooper, mime.c_str(), false /* encoder */);
320
321        CHECK(state->mCodec != NULL);
322
323        err = state->mCodec->configure(
324                format,
325                isVideo ? mSurface : NULL,
326                NULL /* crypto */,
327                0 /* flags */);
328
329        CHECK_EQ(err, (status_t)OK);
330
331        size_t j = 0;
332        sp<ABuffer> buffer;
333        while (format->findBuffer(AStringPrintf("csd-%d", j).c_str(), &buffer)) {
334            state->mCSD.push_back(buffer);
335
336            ++j;
337        }
338    }
339
340    for (size_t i = 0; i < mStateByTrackIndex.size(); ++i) {
341        CodecState *state = &mStateByTrackIndex.editValueAt(i);
342
343        status_t err = state->mCodec->start();
344        CHECK_EQ(err, (status_t)OK);
345
346        err = state->mCodec->getInputBuffers(&state->mBuffers[0]);
347        CHECK_EQ(err, (status_t)OK);
348
349        err = state->mCodec->getOutputBuffers(&state->mBuffers[1]);
350        CHECK_EQ(err, (status_t)OK);
351
352        for (size_t j = 0; j < state->mCSD.size(); ++j) {
353            const sp<ABuffer> &srcBuffer = state->mCSD.itemAt(j);
354
355            size_t index;
356            err = state->mCodec->dequeueInputBuffer(&index, -1ll);
357            CHECK_EQ(err, (status_t)OK);
358
359            const sp<MediaCodecBuffer> &dstBuffer = state->mBuffers[0].itemAt(index);
360
361            CHECK_LE(srcBuffer->size(), dstBuffer->capacity());
362            dstBuffer->setRange(0, srcBuffer->size());
363            memcpy(dstBuffer->data(), srcBuffer->data(), srcBuffer->size());
364
365            err = state->mCodec->queueInputBuffer(
366                    index,
367                    0,
368                    dstBuffer->size(),
369                    0ll,
370                    MediaCodec::BUFFER_FLAG_CODECCONFIG);
371            CHECK_EQ(err, (status_t)OK);
372        }
373    }
374
375    return OK;
376}
377
378status_t SimplePlayer::onStart() {
379    CHECK_EQ(mState, STOPPED);
380
381    mStartTimeRealUs = -1ll;
382
383    sp<AMessage> msg = new AMessage(kWhatDoMoreStuff, this);
384    msg->setInt32("generation", ++mDoMoreStuffGeneration);
385    msg->post();
386
387    return OK;
388}
389
390status_t SimplePlayer::onStop() {
391    CHECK_EQ(mState, STARTED);
392
393    ++mDoMoreStuffGeneration;
394
395    return OK;
396}
397
398status_t SimplePlayer::onReset() {
399    CHECK_EQ(mState, STOPPED);
400
401    for (size_t i = 0; i < mStateByTrackIndex.size(); ++i) {
402        CodecState *state = &mStateByTrackIndex.editValueAt(i);
403
404        CHECK_EQ(state->mCodec->release(), (status_t)OK);
405    }
406
407    mStartTimeRealUs = -1ll;
408
409    mStateByTrackIndex.clear();
410    mCodecLooper.clear();
411    mExtractor.clear();
412    mSurface.clear();
413    mPath.clear();
414
415    return OK;
416}
417
418status_t SimplePlayer::onDoMoreStuff() {
419    ALOGV("onDoMoreStuff");
420    for (size_t i = 0; i < mStateByTrackIndex.size(); ++i) {
421        CodecState *state = &mStateByTrackIndex.editValueAt(i);
422
423        status_t err;
424        do {
425            size_t index;
426            err = state->mCodec->dequeueInputBuffer(&index);
427
428            if (err == OK) {
429                ALOGV("dequeued input buffer on track %zu",
430                      mStateByTrackIndex.keyAt(i));
431
432                state->mAvailInputBufferIndices.push_back(index);
433            } else {
434                ALOGV("dequeueInputBuffer on track %zu returned %d",
435                      mStateByTrackIndex.keyAt(i), err);
436            }
437        } while (err == OK);
438
439        do {
440            BufferInfo info;
441            err = state->mCodec->dequeueOutputBuffer(
442                    &info.mIndex,
443                    &info.mOffset,
444                    &info.mSize,
445                    &info.mPresentationTimeUs,
446                    &info.mFlags);
447
448            if (err == OK) {
449                ALOGV("dequeued output buffer on track %zu",
450                      mStateByTrackIndex.keyAt(i));
451
452                state->mAvailOutputBufferInfos.push_back(info);
453            } else if (err == INFO_FORMAT_CHANGED) {
454                err = onOutputFormatChanged(mStateByTrackIndex.keyAt(i), state);
455                CHECK_EQ(err, (status_t)OK);
456            } else if (err == INFO_OUTPUT_BUFFERS_CHANGED) {
457                err = state->mCodec->getOutputBuffers(&state->mBuffers[1]);
458                CHECK_EQ(err, (status_t)OK);
459            } else {
460                ALOGV("dequeueOutputBuffer on track %zu returned %d",
461                      mStateByTrackIndex.keyAt(i), err);
462            }
463        } while (err == OK
464                || err == INFO_FORMAT_CHANGED
465                || err == INFO_OUTPUT_BUFFERS_CHANGED);
466    }
467
468    for (;;) {
469        size_t trackIndex;
470        status_t err = mExtractor->getSampleTrackIndex(&trackIndex);
471
472        if (err != OK) {
473            ALOGI("encountered input EOS.");
474            break;
475        } else {
476            CodecState *state = &mStateByTrackIndex.editValueFor(trackIndex);
477
478            if (state->mAvailInputBufferIndices.empty()) {
479                break;
480            }
481
482            size_t index = *state->mAvailInputBufferIndices.begin();
483            state->mAvailInputBufferIndices.erase(
484                    state->mAvailInputBufferIndices.begin());
485
486            const sp<MediaCodecBuffer> &dstBuffer =
487                state->mBuffers[0].itemAt(index);
488            sp<ABuffer> abuffer = new ABuffer(dstBuffer->base(), dstBuffer->capacity());
489
490            err = mExtractor->readSampleData(abuffer);
491            CHECK_EQ(err, (status_t)OK);
492            dstBuffer->setRange(abuffer->offset(), abuffer->size());
493
494            int64_t timeUs;
495            CHECK_EQ(mExtractor->getSampleTime(&timeUs), (status_t)OK);
496
497            err = state->mCodec->queueInputBuffer(
498                    index,
499                    dstBuffer->offset(),
500                    dstBuffer->size(),
501                    timeUs,
502                    0);
503            CHECK_EQ(err, (status_t)OK);
504
505            ALOGV("enqueued input data on track %zu", trackIndex);
506
507            err = mExtractor->advance();
508            CHECK_EQ(err, (status_t)OK);
509        }
510    }
511
512    int64_t nowUs = ALooper::GetNowUs();
513
514    if (mStartTimeRealUs < 0ll) {
515        mStartTimeRealUs = nowUs + 1000000ll;
516    }
517
518    for (size_t i = 0; i < mStateByTrackIndex.size(); ++i) {
519        CodecState *state = &mStateByTrackIndex.editValueAt(i);
520
521        while (!state->mAvailOutputBufferInfos.empty()) {
522            BufferInfo *info = &*state->mAvailOutputBufferInfos.begin();
523
524            int64_t whenRealUs = info->mPresentationTimeUs + mStartTimeRealUs;
525            int64_t lateByUs = nowUs - whenRealUs;
526
527            if (lateByUs > -10000ll) {
528                bool release = true;
529
530                if (lateByUs > 30000ll) {
531                    ALOGI("track %zu buffer late by %lld us, dropping.",
532                          mStateByTrackIndex.keyAt(i), (long long)lateByUs);
533                    state->mCodec->releaseOutputBuffer(info->mIndex);
534                } else {
535                    if (state->mAudioTrack != NULL) {
536                        const sp<MediaCodecBuffer> &srcBuffer =
537                            state->mBuffers[1].itemAt(info->mIndex);
538
539                        renderAudio(state, info, srcBuffer);
540
541                        if (info->mSize > 0) {
542                            release = false;
543                        }
544                    }
545
546                    if (release) {
547                        state->mCodec->renderOutputBufferAndRelease(
548                                info->mIndex);
549                    }
550                }
551
552                if (release) {
553                    state->mAvailOutputBufferInfos.erase(
554                            state->mAvailOutputBufferInfos.begin());
555
556                    info = NULL;
557                } else {
558                    break;
559                }
560            } else {
561                ALOGV("track %zu buffer early by %lld us.",
562                      mStateByTrackIndex.keyAt(i), (long long)-lateByUs);
563                break;
564            }
565        }
566    }
567
568    return OK;
569}
570
571status_t SimplePlayer::onOutputFormatChanged(
572        size_t trackIndex __unused, CodecState *state) {
573    sp<AMessage> format;
574    status_t err = state->mCodec->getOutputFormat(&format);
575
576    if (err != OK) {
577        return err;
578    }
579
580    AString mime;
581    CHECK(format->findString("mime", &mime));
582
583    if (!strncasecmp(mime.c_str(), "audio/", 6)) {
584        int32_t channelCount;
585        int32_t sampleRate;
586        CHECK(format->findInt32("channel-count", &channelCount));
587        CHECK(format->findInt32("sample-rate", &sampleRate));
588
589        state->mAudioTrack = new AudioTrack(
590                AUDIO_STREAM_MUSIC,
591                sampleRate,
592                AUDIO_FORMAT_PCM_16_BIT,
593                audio_channel_out_mask_from_count(channelCount),
594                0);
595
596        state->mNumFramesWritten = 0;
597    }
598
599    return OK;
600}
601
602void SimplePlayer::renderAudio(
603        CodecState *state, BufferInfo *info, const sp<MediaCodecBuffer> &buffer) {
604    CHECK(state->mAudioTrack != NULL);
605
606    if (state->mAudioTrack->stopped()) {
607        state->mAudioTrack->start();
608    }
609
610    uint32_t numFramesPlayed;
611    CHECK_EQ(state->mAudioTrack->getPosition(&numFramesPlayed), (status_t)OK);
612
613    uint32_t numFramesAvailableToWrite =
614        state->mAudioTrack->frameCount()
615            - (state->mNumFramesWritten - numFramesPlayed);
616
617    size_t numBytesAvailableToWrite =
618        numFramesAvailableToWrite * state->mAudioTrack->frameSize();
619
620    size_t copy = info->mSize;
621    if (copy > numBytesAvailableToWrite) {
622        copy = numBytesAvailableToWrite;
623    }
624
625    if (copy == 0) {
626        return;
627    }
628
629    int64_t startTimeUs = ALooper::GetNowUs();
630
631    ssize_t nbytes = state->mAudioTrack->write(
632            buffer->base() + info->mOffset, copy);
633
634    CHECK_EQ(nbytes, (ssize_t)copy);
635
636    int64_t delayUs = ALooper::GetNowUs() - startTimeUs;
637
638    uint32_t numFramesWritten = nbytes / state->mAudioTrack->frameSize();
639
640    if (delayUs > 2000ll) {
641        ALOGW("AudioTrack::write took %lld us, numFramesAvailableToWrite=%u, "
642              "numFramesWritten=%u",
643              (long long)delayUs, numFramesAvailableToWrite, numFramesWritten);
644    }
645
646    info->mOffset += nbytes;
647    info->mSize -= nbytes;
648
649    state->mNumFramesWritten += numFramesWritten;
650}
651
652}  // namespace android
653