1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "SimplePlayer"
19#include <utils/Log.h>
20
21#include "SimplePlayer.h"
22
23#include <gui/Surface.h>
24#include <media/AudioTrack.h>
25#include <media/ICrypto.h>
26#include <media/IMediaHTTPService.h>
27#include <media/stagefright/foundation/ABuffer.h>
28#include <media/stagefright/foundation/ADebug.h>
29#include <media/stagefright/foundation/AMessage.h>
30#include <media/stagefright/MediaCodec.h>
31#include <media/stagefright/MediaErrors.h>
32#include <media/stagefright/NativeWindowWrapper.h>
33#include <media/stagefright/NuMediaExtractor.h>
34
35namespace android {
36
37SimplePlayer::SimplePlayer()
38    : mState(UNINITIALIZED),
39      mDoMoreStuffGeneration(0),
40      mStartTimeRealUs(-1ll) {
41}
42
43SimplePlayer::~SimplePlayer() {
44}
45
46// static
47status_t PostAndAwaitResponse(
48        const sp<AMessage> &msg, sp<AMessage> *response) {
49    status_t err = msg->postAndAwaitResponse(response);
50
51    if (err != OK) {
52        return err;
53    }
54
55    if (!(*response)->findInt32("err", &err)) {
56        err = OK;
57    }
58
59    return err;
60}
61status_t SimplePlayer::setDataSource(const char *path) {
62    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
63    msg->setString("path", path);
64    sp<AMessage> response;
65    return PostAndAwaitResponse(msg, &response);
66}
67
68status_t SimplePlayer::setSurface(const sp<IGraphicBufferProducer> &bufferProducer) {
69    sp<AMessage> msg = new AMessage(kWhatSetSurface, id());
70
71    sp<Surface> surface;
72    if (bufferProducer != NULL) {
73        surface = new Surface(bufferProducer);
74    }
75
76    msg->setObject(
77            "native-window", new NativeWindowWrapper(surface));
78
79    sp<AMessage> response;
80    return PostAndAwaitResponse(msg, &response);
81}
82
83status_t SimplePlayer::prepare() {
84    sp<AMessage> msg = new AMessage(kWhatPrepare, id());
85    sp<AMessage> response;
86    return PostAndAwaitResponse(msg, &response);
87}
88
89status_t SimplePlayer::start() {
90    sp<AMessage> msg = new AMessage(kWhatStart, id());
91    sp<AMessage> response;
92    return PostAndAwaitResponse(msg, &response);
93}
94
95status_t SimplePlayer::stop() {
96    sp<AMessage> msg = new AMessage(kWhatStop, id());
97    sp<AMessage> response;
98    return PostAndAwaitResponse(msg, &response);
99}
100
101status_t SimplePlayer::reset() {
102    sp<AMessage> msg = new AMessage(kWhatReset, id());
103    sp<AMessage> response;
104    return PostAndAwaitResponse(msg, &response);
105}
106
107void SimplePlayer::onMessageReceived(const sp<AMessage> &msg) {
108    switch (msg->what()) {
109        case kWhatSetDataSource:
110        {
111            status_t err;
112            if (mState != UNINITIALIZED) {
113                err = INVALID_OPERATION;
114            } else {
115                CHECK(msg->findString("path", &mPath));
116                mState = UNPREPARED;
117            }
118
119            uint32_t replyID;
120            CHECK(msg->senderAwaitsResponse(&replyID));
121
122            sp<AMessage> response = new AMessage;
123            response->setInt32("err", err);
124            response->postReply(replyID);
125            break;
126        }
127
128        case kWhatSetSurface:
129        {
130            status_t err;
131            if (mState != UNPREPARED) {
132                err = INVALID_OPERATION;
133            } else {
134                sp<RefBase> obj;
135                CHECK(msg->findObject("native-window", &obj));
136
137                mNativeWindow = static_cast<NativeWindowWrapper *>(obj.get());
138
139                err = OK;
140            }
141
142            uint32_t replyID;
143            CHECK(msg->senderAwaitsResponse(&replyID));
144
145            sp<AMessage> response = new AMessage;
146            response->setInt32("err", err);
147            response->postReply(replyID);
148            break;
149        }
150
151        case kWhatPrepare:
152        {
153            status_t err;
154            if (mState != UNPREPARED) {
155                err = INVALID_OPERATION;
156            } else {
157                err = onPrepare();
158
159                if (err == OK) {
160                    mState = STOPPED;
161                }
162            }
163
164            uint32_t replyID;
165            CHECK(msg->senderAwaitsResponse(&replyID));
166
167            sp<AMessage> response = new AMessage;
168            response->setInt32("err", err);
169            response->postReply(replyID);
170            break;
171        }
172
173        case kWhatStart:
174        {
175            status_t err = OK;
176
177            if (mState == UNPREPARED) {
178                err = onPrepare();
179
180                if (err == OK) {
181                    mState = STOPPED;
182                }
183            }
184
185            if (err == OK) {
186                if (mState != STOPPED) {
187                    err = INVALID_OPERATION;
188                } else {
189                    err = onStart();
190
191                    if (err == OK) {
192                        mState = STARTED;
193                    }
194                }
195            }
196
197            uint32_t replyID;
198            CHECK(msg->senderAwaitsResponse(&replyID));
199
200            sp<AMessage> response = new AMessage;
201            response->setInt32("err", err);
202            response->postReply(replyID);
203            break;
204        }
205
206        case kWhatStop:
207        {
208            status_t err;
209
210            if (mState != STARTED) {
211                err = INVALID_OPERATION;
212            } else {
213                err = onStop();
214
215                if (err == OK) {
216                    mState = STOPPED;
217                }
218            }
219
220            uint32_t replyID;
221            CHECK(msg->senderAwaitsResponse(&replyID));
222
223            sp<AMessage> response = new AMessage;
224            response->setInt32("err", err);
225            response->postReply(replyID);
226            break;
227        }
228
229        case kWhatReset:
230        {
231            status_t err = OK;
232
233            if (mState == STARTED) {
234                CHECK_EQ(onStop(), (status_t)OK);
235                mState = STOPPED;
236            }
237
238            if (mState == STOPPED) {
239                err = onReset();
240                mState = UNINITIALIZED;
241            }
242
243            uint32_t replyID;
244            CHECK(msg->senderAwaitsResponse(&replyID));
245
246            sp<AMessage> response = new AMessage;
247            response->setInt32("err", err);
248            response->postReply(replyID);
249            break;
250        }
251
252        case kWhatDoMoreStuff:
253        {
254            int32_t generation;
255            CHECK(msg->findInt32("generation", &generation));
256
257            if (generation != mDoMoreStuffGeneration) {
258                break;
259            }
260
261            status_t err = onDoMoreStuff();
262
263            if (err == OK) {
264                msg->post(10000ll);
265            }
266            break;
267        }
268
269        default:
270            TRESPASS();
271    }
272}
273
274status_t SimplePlayer::onPrepare() {
275    CHECK_EQ(mState, UNPREPARED);
276
277    mExtractor = new NuMediaExtractor;
278
279    status_t err = mExtractor->setDataSource(
280            NULL /* httpService */, mPath.c_str());
281
282    if (err != OK) {
283        mExtractor.clear();
284        return err;
285    }
286
287    if (mCodecLooper == NULL) {
288        mCodecLooper = new ALooper;
289        mCodecLooper->start();
290    }
291
292    bool haveAudio = false;
293    bool haveVideo = false;
294    for (size_t i = 0; i < mExtractor->countTracks(); ++i) {
295        sp<AMessage> format;
296        status_t err = mExtractor->getTrackFormat(i, &format);
297        CHECK_EQ(err, (status_t)OK);
298
299        AString mime;
300        CHECK(format->findString("mime", &mime));
301
302        bool isVideo = !strncasecmp(mime.c_str(), "video/", 6);
303
304        if (!haveAudio && !strncasecmp(mime.c_str(), "audio/", 6)) {
305            haveAudio = true;
306        } else if (!haveVideo && isVideo) {
307            haveVideo = true;
308        } else {
309            continue;
310        }
311
312        err = mExtractor->selectTrack(i);
313        CHECK_EQ(err, (status_t)OK);
314
315        CodecState *state =
316            &mStateByTrackIndex.editValueAt(
317                    mStateByTrackIndex.add(i, CodecState()));
318
319        state->mNumFramesWritten = 0;
320        state->mCodec = MediaCodec::CreateByType(
321                mCodecLooper, mime.c_str(), false /* encoder */);
322
323        CHECK(state->mCodec != NULL);
324
325        err = state->mCodec->configure(
326                format,
327                isVideo ? mNativeWindow->getSurfaceTextureClient() : NULL,
328                NULL /* crypto */,
329                0 /* flags */);
330
331        CHECK_EQ(err, (status_t)OK);
332
333        size_t j = 0;
334        sp<ABuffer> buffer;
335        while (format->findBuffer(StringPrintf("csd-%d", j).c_str(), &buffer)) {
336            state->mCSD.push_back(buffer);
337
338            ++j;
339        }
340    }
341
342    for (size_t i = 0; i < mStateByTrackIndex.size(); ++i) {
343        CodecState *state = &mStateByTrackIndex.editValueAt(i);
344
345        status_t err = state->mCodec->start();
346        CHECK_EQ(err, (status_t)OK);
347
348        err = state->mCodec->getInputBuffers(&state->mBuffers[0]);
349        CHECK_EQ(err, (status_t)OK);
350
351        err = state->mCodec->getOutputBuffers(&state->mBuffers[1]);
352        CHECK_EQ(err, (status_t)OK);
353
354        for (size_t j = 0; j < state->mCSD.size(); ++j) {
355            const sp<ABuffer> &srcBuffer = state->mCSD.itemAt(j);
356
357            size_t index;
358            err = state->mCodec->dequeueInputBuffer(&index, -1ll);
359            CHECK_EQ(err, (status_t)OK);
360
361            const sp<ABuffer> &dstBuffer = state->mBuffers[0].itemAt(index);
362
363            CHECK_LE(srcBuffer->size(), dstBuffer->capacity());
364            dstBuffer->setRange(0, srcBuffer->size());
365            memcpy(dstBuffer->data(), srcBuffer->data(), srcBuffer->size());
366
367            err = state->mCodec->queueInputBuffer(
368                    index,
369                    0,
370                    dstBuffer->size(),
371                    0ll,
372                    MediaCodec::BUFFER_FLAG_CODECCONFIG);
373            CHECK_EQ(err, (status_t)OK);
374        }
375    }
376
377    return OK;
378}
379
380status_t SimplePlayer::onStart() {
381    CHECK_EQ(mState, STOPPED);
382
383    mStartTimeRealUs = -1ll;
384
385    sp<AMessage> msg = new AMessage(kWhatDoMoreStuff, id());
386    msg->setInt32("generation", ++mDoMoreStuffGeneration);
387    msg->post();
388
389    return OK;
390}
391
392status_t SimplePlayer::onStop() {
393    CHECK_EQ(mState, STARTED);
394
395    ++mDoMoreStuffGeneration;
396
397    return OK;
398}
399
400status_t SimplePlayer::onReset() {
401    CHECK_EQ(mState, STOPPED);
402
403    for (size_t i = 0; i < mStateByTrackIndex.size(); ++i) {
404        CodecState *state = &mStateByTrackIndex.editValueAt(i);
405
406        CHECK_EQ(state->mCodec->release(), (status_t)OK);
407    }
408
409    mStartTimeRealUs = -1ll;
410
411    mStateByTrackIndex.clear();
412    mCodecLooper.clear();
413    mExtractor.clear();
414    mNativeWindow.clear();
415    mPath.clear();
416
417    return OK;
418}
419
420status_t SimplePlayer::onDoMoreStuff() {
421    ALOGV("onDoMoreStuff");
422    for (size_t i = 0; i < mStateByTrackIndex.size(); ++i) {
423        CodecState *state = &mStateByTrackIndex.editValueAt(i);
424
425        status_t err;
426        do {
427            size_t index;
428            err = state->mCodec->dequeueInputBuffer(&index);
429
430            if (err == OK) {
431                ALOGV("dequeued input buffer on track %d",
432                      mStateByTrackIndex.keyAt(i));
433
434                state->mAvailInputBufferIndices.push_back(index);
435            } else {
436                ALOGV("dequeueInputBuffer on track %d returned %d",
437                      mStateByTrackIndex.keyAt(i), err);
438            }
439        } while (err == OK);
440
441        do {
442            BufferInfo info;
443            err = state->mCodec->dequeueOutputBuffer(
444                    &info.mIndex,
445                    &info.mOffset,
446                    &info.mSize,
447                    &info.mPresentationTimeUs,
448                    &info.mFlags);
449
450            if (err == OK) {
451                ALOGV("dequeued output buffer on track %d",
452                      mStateByTrackIndex.keyAt(i));
453
454                state->mAvailOutputBufferInfos.push_back(info);
455            } else if (err == INFO_FORMAT_CHANGED) {
456                err = onOutputFormatChanged(mStateByTrackIndex.keyAt(i), state);
457                CHECK_EQ(err, (status_t)OK);
458            } else if (err == INFO_OUTPUT_BUFFERS_CHANGED) {
459                err = state->mCodec->getOutputBuffers(&state->mBuffers[1]);
460                CHECK_EQ(err, (status_t)OK);
461            } else {
462                ALOGV("dequeueOutputBuffer on track %d returned %d",
463                      mStateByTrackIndex.keyAt(i), err);
464            }
465        } while (err == OK
466                || err == INFO_FORMAT_CHANGED
467                || err == INFO_OUTPUT_BUFFERS_CHANGED);
468    }
469
470    for (;;) {
471        size_t trackIndex;
472        status_t err = mExtractor->getSampleTrackIndex(&trackIndex);
473
474        if (err != OK) {
475            ALOGI("encountered input EOS.");
476            break;
477        } else {
478            CodecState *state = &mStateByTrackIndex.editValueFor(trackIndex);
479
480            if (state->mAvailInputBufferIndices.empty()) {
481                break;
482            }
483
484            size_t index = *state->mAvailInputBufferIndices.begin();
485            state->mAvailInputBufferIndices.erase(
486                    state->mAvailInputBufferIndices.begin());
487
488            const sp<ABuffer> &dstBuffer =
489                state->mBuffers[0].itemAt(index);
490
491            err = mExtractor->readSampleData(dstBuffer);
492            CHECK_EQ(err, (status_t)OK);
493
494            int64_t timeUs;
495            CHECK_EQ(mExtractor->getSampleTime(&timeUs), (status_t)OK);
496
497            err = state->mCodec->queueInputBuffer(
498                    index,
499                    dstBuffer->offset(),
500                    dstBuffer->size(),
501                    timeUs,
502                    0);
503            CHECK_EQ(err, (status_t)OK);
504
505            ALOGV("enqueued input data on track %d", trackIndex);
506
507            err = mExtractor->advance();
508            CHECK_EQ(err, (status_t)OK);
509        }
510    }
511
512    int64_t nowUs = ALooper::GetNowUs();
513
514    if (mStartTimeRealUs < 0ll) {
515        mStartTimeRealUs = nowUs + 1000000ll;
516    }
517
518    for (size_t i = 0; i < mStateByTrackIndex.size(); ++i) {
519        CodecState *state = &mStateByTrackIndex.editValueAt(i);
520
521        while (!state->mAvailOutputBufferInfos.empty()) {
522            BufferInfo *info = &*state->mAvailOutputBufferInfos.begin();
523
524            int64_t whenRealUs = info->mPresentationTimeUs + mStartTimeRealUs;
525            int64_t lateByUs = nowUs - whenRealUs;
526
527            if (lateByUs > -10000ll) {
528                bool release = true;
529
530                if (lateByUs > 30000ll) {
531                    ALOGI("track %d buffer late by %lld us, dropping.",
532                          mStateByTrackIndex.keyAt(i), lateByUs);
533                    state->mCodec->releaseOutputBuffer(info->mIndex);
534                } else {
535                    if (state->mAudioTrack != NULL) {
536                        const sp<ABuffer> &srcBuffer =
537                            state->mBuffers[1].itemAt(info->mIndex);
538
539                        renderAudio(state, info, srcBuffer);
540
541                        if (info->mSize > 0) {
542                            release = false;
543                        }
544                    }
545
546                    if (release) {
547                        state->mCodec->renderOutputBufferAndRelease(
548                                info->mIndex);
549                    }
550                }
551
552                if (release) {
553                    state->mAvailOutputBufferInfos.erase(
554                            state->mAvailOutputBufferInfos.begin());
555
556                    info = NULL;
557                } else {
558                    break;
559                }
560            } else {
561                ALOGV("track %d buffer early by %lld us.",
562                      mStateByTrackIndex.keyAt(i), -lateByUs);
563                break;
564            }
565        }
566    }
567
568    return OK;
569}
570
571status_t SimplePlayer::onOutputFormatChanged(
572        size_t trackIndex, CodecState *state) {
573    sp<AMessage> format;
574    status_t err = state->mCodec->getOutputFormat(&format);
575
576    if (err != OK) {
577        return err;
578    }
579
580    AString mime;
581    CHECK(format->findString("mime", &mime));
582
583    if (!strncasecmp(mime.c_str(), "audio/", 6)) {
584        int32_t channelCount;
585        int32_t sampleRate;
586        CHECK(format->findInt32("channel-count", &channelCount));
587        CHECK(format->findInt32("sample-rate", &sampleRate));
588
589        state->mAudioTrack = new AudioTrack(
590                AUDIO_STREAM_MUSIC,
591                sampleRate,
592                AUDIO_FORMAT_PCM_16_BIT,
593                audio_channel_out_mask_from_count(channelCount),
594                0);
595
596        state->mNumFramesWritten = 0;
597    }
598
599    return OK;
600}
601
602void SimplePlayer::renderAudio(
603        CodecState *state, BufferInfo *info, const sp<ABuffer> &buffer) {
604    CHECK(state->mAudioTrack != NULL);
605
606    if (state->mAudioTrack->stopped()) {
607        state->mAudioTrack->start();
608    }
609
610    uint32_t numFramesPlayed;
611    CHECK_EQ(state->mAudioTrack->getPosition(&numFramesPlayed), (status_t)OK);
612
613    uint32_t numFramesAvailableToWrite =
614        state->mAudioTrack->frameCount()
615            - (state->mNumFramesWritten - numFramesPlayed);
616
617    size_t numBytesAvailableToWrite =
618        numFramesAvailableToWrite * state->mAudioTrack->frameSize();
619
620    size_t copy = info->mSize;
621    if (copy > numBytesAvailableToWrite) {
622        copy = numBytesAvailableToWrite;
623    }
624
625    if (copy == 0) {
626        return;
627    }
628
629    int64_t startTimeUs = ALooper::GetNowUs();
630
631    ssize_t nbytes = state->mAudioTrack->write(
632            buffer->base() + info->mOffset, copy);
633
634    CHECK_EQ(nbytes, (ssize_t)copy);
635
636    int64_t delayUs = ALooper::GetNowUs() - startTimeUs;
637
638    uint32_t numFramesWritten = nbytes / state->mAudioTrack->frameSize();
639
640    if (delayUs > 2000ll) {
641        ALOGW("AudioTrack::write took %lld us, numFramesAvailableToWrite=%u, "
642              "numFramesWritten=%u",
643              delayUs, numFramesAvailableToWrite, numFramesWritten);
644    }
645
646    info->mOffset += nbytes;
647    info->mSize -= nbytes;
648
649    state->mNumFramesWritten += numFramesWritten;
650}
651
652}  // namespace android
653