1c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia/* 2c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * Copyright 2015 The Android Open Source Project 3c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * 4c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * Licensed under the Apache License, Version 2.0 (the "License"); 5c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * you may not use this file except in compliance with the License. 6c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * You may obtain a copy of the License at 7c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * 8c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * http://www.apache.org/licenses/LICENSE-2.0 9c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * 10c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * Unless required by applicable law or agreed to in writing, software 11c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * distributed under the License is distributed on an "AS IS" BASIS, 12c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * See the License for the specific language governing permissions and 14c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * limitations under the License. 15c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia */ 16c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 17c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia//#define LOG_NDEBUG 0 18c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#define LOG_TAG "MediaSync" 19c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <inttypes.h> 20c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 21c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <gui/BufferQueue.h> 22c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <gui/IGraphicBufferConsumer.h> 23c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <gui/IGraphicBufferProducer.h> 24c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 25c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <media/AudioTrack.h> 26c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <media/stagefright/MediaClock.h> 27c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <media/stagefright/MediaSync.h> 28a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar#include <media/stagefright/VideoFrameScheduler.h> 29c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <media/stagefright/foundation/ADebug.h> 30c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <media/stagefright/foundation/ALooper.h> 31c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <media/stagefright/foundation/AMessage.h> 32c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 33c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <ui/GraphicBuffer.h> 34c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 35defb1b0b7e082621a10763d1bd7a4a01e280fdf0Mathias Agopian#include <system/window.h> 36defb1b0b7e082621a10763d1bd7a4a01e280fdf0Mathias Agopian 37c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia// Maximum late time allowed for a video frame to be rendered. When a video 38c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia// frame arrives later than this number, it will be discarded without rendering. 39c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiastatic const int64_t kMaxAllowedVideoLateTimeUs = 40000ll; 40c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 41c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jianamespace android { 42c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 43c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia// static 44c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiasp<MediaSync> MediaSync::create() { 45c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<MediaSync> sync = new MediaSync(); 46c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sync->mLooper->registerHandler(sync); 47c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return sync; 48c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 49c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 50c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei JiaMediaSync::MediaSync() 51c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia : mIsAbandoned(false), 52c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mMutex(), 53c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mReleaseCondition(), 54c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mNumOutstandingBuffers(0), 55b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mUsageFlagsFromOutput(0), 56a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mMaxAcquiredBufferCount(1), 5708dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mReturnPendingInputFrame(false), 58c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mNativeSampleRateInHz(0), 59c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mNumFramesWritten(0), 60c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mHasAudio(false), 61c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mNextBufferItemMediaUs(-1), 62c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mPlaybackRate(0.0) { 63c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mMediaClock = new MediaClock; 647b6f67d18a4a4e94fddb836de1176b609968643aWei Jia mMediaClock->init(); 65c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 663a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // initialize settings 673a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT; 683a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mPlaybackSettings.mSpeed = mPlaybackRate; 693a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 70c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mLooper = new ALooper; 71c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mLooper->setName("MediaSync"); 72c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 73c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 74c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 75c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei JiaMediaSync::~MediaSync() { 76c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mInput != NULL) { 77c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mInput->consumerDisconnect(); 78c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 79c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mOutput != NULL) { 80c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 81c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 82c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 83c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mLooper != NULL) { 84c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mLooper->unregisterHandler(id()); 85c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mLooper->stop(); 86c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 87c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 88c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 89ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jiastatus_t MediaSync::setSurface(const sp<IGraphicBufferProducer> &output) { 90c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 91c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 92b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia if (output == mOutput) { 93b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia return NO_ERROR; // same output surface. 94c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 95c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 963a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (output == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_VSYNC) { 97ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jia ALOGE("setSurface: output surface is used as sync source and cannot be removed."); 983a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return INVALID_OPERATION; 993a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 1003a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 101c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (output != NULL) { 102b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia int newUsage = 0; 103b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia output->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &newUsage); 104b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia 105b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia // Check usage flags only when current output surface has been used to create input surface. 106b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia if (mOutput != NULL && mInput != NULL) { 107b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia int ignoredFlags = (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_COMPOSER 108b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia | GRALLOC_USAGE_EXTERNAL_DISP); 109b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia // New output surface is not allowed to add new usage flag except ignored ones. 110b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia if ((newUsage & ~(mUsageFlagsFromOutput | ignoredFlags)) != 0) { 111b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia ALOGE("setSurface: new output surface has new usage flag not used by current one."); 112b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia return BAD_VALUE; 113b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia } 114b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia } 115b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia 116b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia // Try to connect to new output surface. If failed, current output surface will not 117b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia // be changed. 118c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia IGraphicBufferProducer::QueueBufferOutput queueBufferOutput; 119b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia sp<OutputListener> listener(new OutputListener(this, output)); 120c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia IInterface::asBinder(output)->linkToDeath(listener); 121c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status_t status = 122c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia output->connect(listener, 123c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia NATIVE_WINDOW_API_MEDIA, 124c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia true /* producerControlledByApp */, 125c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia &queueBufferOutput); 126c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status != NO_ERROR) { 127ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jia ALOGE("setSurface: failed to connect (%d)", status); 128c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return status; 129c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 130a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar 131a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (mFrameScheduler == NULL) { 132a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mFrameScheduler = new VideoFrameScheduler(); 133a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mFrameScheduler->init(); 134a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 135b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia } 136c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 137b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia if (mOutput != NULL) { 138b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 139b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia while (!mBuffersSentToOutput.isEmpty()) { 140b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia returnBufferToInput_l(mBuffersSentToOutput.valueAt(0), Fence::NO_FENCE); 141b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mBuffersSentToOutput.removeItemsAt(0); 142b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia } 143c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 144c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 145b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mOutput = output; 146b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia 147c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return NO_ERROR; 148c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 149c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 150c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia// |audioTrack| is used only for querying information. 151ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jiastatus_t MediaSync::setAudioTrack(const sp<AudioTrack> &audioTrack) { 152c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 153c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 154c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // TODO: support audio track change. 155c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mAudioTrack != NULL) { 156ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jia ALOGE("setAudioTrack: audioTrack has already been configured."); 157c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return INVALID_OPERATION; 158c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 159c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 1603a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (audioTrack == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_AUDIO) { 161ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jia ALOGE("setAudioTrack: audioTrack is used as sync source and cannot be removed."); 1623a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return INVALID_OPERATION; 1639767a47dcf9bbe090045ab99e0290d73289c704fWei Jia } 1649767a47dcf9bbe090045ab99e0290d73289c704fWei Jia 1653a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (audioTrack != NULL) { 1663a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // check if audio track supports the playback settings 1673a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (mPlaybackSettings.mSpeed != 0.f 1683a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar && audioTrack->setPlaybackRate(mPlaybackSettings) != OK) { 1693a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar ALOGE("playback settings are not supported by the audio track"); 1703a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return INVALID_OPERATION; 1713a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 1723a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar uint32_t nativeSampleRateInHz = audioTrack->getOriginalSampleRate(); 1733a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (nativeSampleRateInHz <= 0) { 174ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jia ALOGE("setAudioTrack: native sample rate should be positive."); 1753a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return BAD_VALUE; 1763a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 1773a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mAudioTrack = audioTrack; 1783a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mNativeSampleRateInHz = nativeSampleRateInHz; 1793a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar (void)setPlaybackSettings_l(mPlaybackSettings); 1803a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 1813a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar else { 1823a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mAudioTrack = NULL; 1833a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mNativeSampleRateInHz = 0; 1843a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 185c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 1863a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // potentially resync to new source 1873a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar resync_l(); 1883a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return OK; 189c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 190c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 191c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiastatus_t MediaSync::createInputSurface( 192c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<IGraphicBufferProducer> *outBufferProducer) { 193c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (outBufferProducer == NULL) { 194c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return BAD_VALUE; 195c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 196c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 197c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 198c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 199c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mOutput == NULL) { 200c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return NO_INIT; 201c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 202c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 203c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mInput != NULL) { 204c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return INVALID_OPERATION; 205c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 206c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 207c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<IGraphicBufferProducer> bufferProducer; 208c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<IGraphicBufferConsumer> bufferConsumer; 209c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia BufferQueue::createBufferQueue(&bufferProducer, &bufferConsumer); 210c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 211c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<InputListener> listener(new InputListener(this)); 212c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia IInterface::asBinder(bufferConsumer)->linkToDeath(listener); 213c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status_t status = 214c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferConsumer->consumerConnect(listener, false /* controlledByApp */); 215c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status == NO_ERROR) { 216c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferConsumer->setConsumerName(String8("MediaSync")); 2179b73fa9a02b7234b43ac2be66a3328ff36d39c2dLajos Molnar // propagate usage bits from output surface 218b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mUsageFlagsFromOutput = 0; 219b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mOutput->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &mUsageFlagsFromOutput); 220b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia bufferConsumer->setConsumerUsageBits(mUsageFlagsFromOutput); 221c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia *outBufferProducer = bufferProducer; 222c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mInput = bufferConsumer; 223a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar 224a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar // set undequeued buffer count 225a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar int minUndequeuedBuffers; 226a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mOutput->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers); 227a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mMaxAcquiredBufferCount = minUndequeuedBuffers; 228a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar bufferConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount); 229c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 230c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return status; 231c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 232c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 2333a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarvoid MediaSync::resync_l() { 2343a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar AVSyncSource src = mSyncSettings.mSource; 2353a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (src == AVSYNC_SOURCE_DEFAULT) { 2363a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (mAudioTrack != NULL) { 2373a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar src = AVSYNC_SOURCE_AUDIO; 2383a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } else { 2393a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar src = AVSYNC_SOURCE_SYSTEM_CLOCK; 2403a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 241c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 242c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 2433a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // TODO: resync ourselves to the current clock (e.g. on sync source change) 2443a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar updatePlaybackRate_l(mPlaybackRate); 2453a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 246c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 2473a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarvoid MediaSync::updatePlaybackRate_l(float rate) { 248c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (rate > mPlaybackRate) { 249c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mNextBufferItemMediaUs = -1; 250c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 251c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mPlaybackRate = rate; 25208dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar // TODO: update frame scheduler with this info 253c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mMediaClock->setPlaybackRate(rate); 254c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia onDrainVideo_l(); 255c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 256c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 257c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiasp<const MediaClock> MediaSync::getMediaClock() { 258c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return mMediaClock; 259c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 260c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 26199d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jiastatus_t MediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) { 26299d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia Mutex::Autolock lock(mMutex); 26399d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia // User should check the playback rate if it doesn't want to receive a 26499d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia // huge number for play time. 26599d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia if (mPlaybackRate == 0.0f) { 26699d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia *outTimeUs = INT64_MAX; 26799d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia return OK; 26899d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia } 26999d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia 27099d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia uint32_t numFramesPlayed = 0; 27199d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia if (mAudioTrack != NULL) { 27299d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia status_t res = mAudioTrack->getPosition(&numFramesPlayed); 27399d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia if (res != OK) { 27499d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia return res; 27599d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia } 27699d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia } 27799d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia 27899d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia int64_t numPendingFrames = mNumFramesWritten - numFramesPlayed; 27999d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia if (numPendingFrames < 0) { 28099d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia numPendingFrames = 0; 28199d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia ALOGW("getPlayTimeForPendingAudioFrames: pending frame count is negative."); 28299d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia } 28399d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia double timeUs = numPendingFrames * 1000000.0 28499d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia / (mNativeSampleRateInHz * (double)mPlaybackRate); 28599d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia if (timeUs > (double)INT64_MAX) { 28699d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia // Overflow. 28799d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia *outTimeUs = INT64_MAX; 28899d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia ALOGW("getPlayTimeForPendingAudioFrames: play time for pending audio frames " 28999d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia "is too high, possibly due to super low playback rate(%f)", mPlaybackRate); 29099d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia } else { 29199d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia *outTimeUs = (int64_t)timeUs; 29299d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia } 29399d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia 29499d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia return OK; 29599d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia} 29699d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia 297c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiastatus_t MediaSync::updateQueuedAudioData( 298c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia size_t sizeInBytes, int64_t presentationTimeUs) { 299c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (sizeInBytes == 0) { 300c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return OK; 301c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 302c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 303c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 304c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 305c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mAudioTrack == NULL) { 306c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGW("updateQueuedAudioData: audioTrack has NOT been configured."); 307c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return INVALID_OPERATION; 308c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 309c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 310c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t numFrames = sizeInBytes / mAudioTrack->frameSize(); 311c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t maxMediaTimeUs = presentationTimeUs 312c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia + getDurationIfPlayedAtNativeSampleRate_l(numFrames); 313c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 314c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t nowUs = ALooper::GetNowUs(); 31580243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia int64_t nowMediaUs = presentationTimeUs 316c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia - getDurationIfPlayedAtNativeSampleRate_l(mNumFramesWritten) 317c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia + getPlayedOutAudioDurationMedia_l(nowUs); 318c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 31980243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia mNumFramesWritten += numFrames; 32080243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia 321c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t oldRealTime = -1; 322c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mNextBufferItemMediaUs != -1) { 323c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia oldRealTime = getRealTime(mNextBufferItemMediaUs, nowUs); 324c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 325c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 326c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mMediaClock->updateAnchor(nowMediaUs, nowUs, maxMediaTimeUs); 327c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mHasAudio = true; 328c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 329c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (oldRealTime != -1) { 330c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t newRealTime = getRealTime(mNextBufferItemMediaUs, nowUs); 33180243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia if (newRealTime >= oldRealTime) { 33280243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia return OK; 333c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 334c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 335c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 33680243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia mNextBufferItemMediaUs = -1; 33780243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia onDrainVideo_l(); 338c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return OK; 339c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 340c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 341c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::setName(const AString &name) { 342c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 343c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mInput->setConsumerName(String8(name.c_str())); 344c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 345c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 34608dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnarvoid MediaSync::flush() { 34708dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar Mutex::Autolock lock(mMutex); 34808dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar if (mFrameScheduler != NULL) { 34908dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mFrameScheduler->restart(); 35008dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar } 35108dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar while (!mBufferItems.empty()) { 35208dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar BufferItem *bufferItem = &*mBufferItems.begin(); 35308dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar returnBufferToInput_l(bufferItem->mGraphicBuffer, bufferItem->mFence); 35408dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mBufferItems.erase(mBufferItems.begin()); 35508dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar } 35608dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mNextBufferItemMediaUs = -1; 35708dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mNumFramesWritten = 0; 35808dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mReturnPendingInputFrame = true; 35908dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mReleaseCondition.signal(); 36008dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mMediaClock->clearAnchor(); 36108dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar} 36208dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar 3633a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarstatus_t MediaSync::setVideoFrameRateHint(float rate) { 364a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar Mutex::Autolock lock(mMutex); 365a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (rate < 0.f) { 366a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar return BAD_VALUE; 367a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 368a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (mFrameScheduler != NULL) { 369a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mFrameScheduler->init(rate); 370a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 371a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar return OK; 3723a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 3733a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 3743a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarfloat MediaSync::getVideoFrameRate() { 375a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar Mutex::Autolock lock(mMutex); 376a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (mFrameScheduler != NULL) { 377a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar float fps = mFrameScheduler->getFrameRate(); 378a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (fps > 0.f) { 379a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar return fps; 380a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 381a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 382a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar 383a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar // we don't have or know the frame rate 3843a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return -1.f; 3853a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 3863a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 3873a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarstatus_t MediaSync::setSyncSettings(const AVSyncSettings &syncSettings) { 3883a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // validate settings 3893a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (syncSettings.mSource >= AVSYNC_SOURCE_MAX 3903a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar || syncSettings.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX 3913a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar || syncSettings.mTolerance < 0.f 3923a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar || syncSettings.mTolerance >= AVSYNC_TOLERANCE_MAX) { 3933a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return BAD_VALUE; 3943a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 3953a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 3963a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar Mutex::Autolock lock(mMutex); 3973a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 3983a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // verify that we have the sync source 3993a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar switch (syncSettings.mSource) { 4003a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar case AVSYNC_SOURCE_AUDIO: 4013a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (mAudioTrack == NULL) { 4023a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar ALOGE("setSyncSettings: audio sync source requires an audio track"); 4033a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return BAD_VALUE; 4043a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4053a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar break; 4063a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar case AVSYNC_SOURCE_VSYNC: 4073a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (mOutput == NULL) { 4083a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar ALOGE("setSyncSettings: vsync sync source requires an output surface"); 4093a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return BAD_VALUE; 4103a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4113a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar break; 4123a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar default: 4133a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar break; 4143a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4153a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4163a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mSyncSettings = syncSettings; 4173a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar resync_l(); 4183a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return OK; 4193a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 4203a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4213a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarvoid MediaSync::getSyncSettings(AVSyncSettings *syncSettings) { 4223a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar Mutex::Autolock lock(mMutex); 4233a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar *syncSettings = mSyncSettings; 4243a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 4253a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4263a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarstatus_t MediaSync::setPlaybackSettings(const AudioPlaybackRate &rate) { 4273a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar Mutex::Autolock lock(mMutex); 4283a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4293a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar status_t err = setPlaybackSettings_l(rate); 4303a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (err == OK) { 4313a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // TODO: adjust rate if using VSYNC as source 4323a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar updatePlaybackRate_l(rate.mSpeed); 4333a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4343a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return err; 4353a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 4363a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4373a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarstatus_t MediaSync::setPlaybackSettings_l(const AudioPlaybackRate &rate) { 4383a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (rate.mSpeed < 0.f || rate.mPitch < 0.f) { 4393a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // We don't validate other audio settings. 4403a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // They will be validated when/if audiotrack is set. 4413a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return BAD_VALUE; 4423a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4433a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4443a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (mAudioTrack != NULL) { 4453a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (rate.mSpeed == 0.f) { 4463a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mAudioTrack->pause(); 4473a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } else { 4483a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar status_t err = mAudioTrack->setPlaybackRate(rate); 4493a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (err != OK) { 4503a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return BAD_VALUE; 4513a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4523a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4533a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // ignore errors 4543a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar (void)mAudioTrack->start(); 4553a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4563a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4573a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mPlaybackSettings = rate; 4583a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return OK; 4593a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 4603a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4613a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarvoid MediaSync::getPlaybackSettings(AudioPlaybackRate *rate) { 4623a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar Mutex::Autolock lock(mMutex); 4633a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar *rate = mPlaybackSettings; 4643a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 4653a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 466c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiaint64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) { 467c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t realUs; 468c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) { 469c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // If failed to get current position, e.g. due to audio clock is 470c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // not ready, then just play out video immediately without delay. 471c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return nowUs; 472c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 473c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return realUs; 474c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 475c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 476c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiaint64_t MediaSync::getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames) { 477c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return (numFrames * 1000000LL / mNativeSampleRateInHz); 478c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 479c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 480c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiaint64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) { 481c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia CHECK(mAudioTrack != NULL); 482c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 483c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia uint32_t numFramesPlayed; 4845d313803bdd61d59f28e518e7a793f33832f1901Andy Hung int64_t numFramesPlayedAtUs; 485c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia AudioTimestamp ts; 486c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 487c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status_t res = mAudioTrack->getTimestamp(ts); 488c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (res == OK) { 489c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // case 1: mixing audio tracks. 490c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia numFramesPlayed = ts.mPosition; 4915d313803bdd61d59f28e518e7a793f33832f1901Andy Hung numFramesPlayedAtUs = ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000; 492c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia //ALOGD("getTimestamp: OK %d %lld", 4935d313803bdd61d59f28e518e7a793f33832f1901Andy Hung // numFramesPlayed, (long long)numFramesPlayedAtUs); 494c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } else if (res == WOULD_BLOCK) { 495c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // case 2: transitory state on start of a new track 496c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia numFramesPlayed = 0; 4975d313803bdd61d59f28e518e7a793f33832f1901Andy Hung numFramesPlayedAtUs = nowUs; 498c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia //ALOGD("getTimestamp: WOULD_BLOCK %d %lld", 4995d313803bdd61d59f28e518e7a793f33832f1901Andy Hung // numFramesPlayed, (long long)numFramesPlayedAtUs); 500c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } else { 501c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // case 3: transitory at new track or audio fast tracks. 502c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia res = mAudioTrack->getPosition(&numFramesPlayed); 503c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia CHECK_EQ(res, (status_t)OK); 5045d313803bdd61d59f28e518e7a793f33832f1901Andy Hung numFramesPlayedAtUs = nowUs; 5055d313803bdd61d59f28e518e7a793f33832f1901Andy Hung numFramesPlayedAtUs += 1000LL * mAudioTrack->latency() / 2; /* XXX */ 5065d313803bdd61d59f28e518e7a793f33832f1901Andy Hung //ALOGD("getPosition: %d %lld", numFramesPlayed, (long long)numFramesPlayedAtUs); 507c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 508c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 509c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia //can't be negative until 12.4 hrs, test. 510c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia //CHECK_EQ(numFramesPlayed & (1 << 31), 0); 511c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t durationUs = 512c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia getDurationIfPlayedAtNativeSampleRate_l(numFramesPlayed) 5135d313803bdd61d59f28e518e7a793f33832f1901Andy Hung + nowUs - numFramesPlayedAtUs; 514c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (durationUs < 0) { 515c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // Occurs when numFramesPlayed position is very small and the following: 516c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // (1) In case 1, the time nowUs is computed before getTimestamp() is 5175d313803bdd61d59f28e518e7a793f33832f1901Andy Hung // called and numFramesPlayedAtUs is greater than nowUs by time more 518c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // than numFramesPlayed. 519c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // (2) In case 3, using getPosition and adding mAudioTrack->latency() 5205d313803bdd61d59f28e518e7a793f33832f1901Andy Hung // to numFramesPlayedAtUs, by a time amount greater than 521c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // numFramesPlayed. 522c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // 523c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // Both of these are transitory conditions. 524c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGV("getPlayedOutAudioDurationMedia_l: negative duration %lld " 525c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia "set to zero", (long long)durationUs); 526c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia durationUs = 0; 527c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 528c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGV("getPlayedOutAudioDurationMedia_l(%lld) nowUs(%lld) frames(%u) " 529c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia "framesAt(%lld)", 530c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia (long long)durationUs, (long long)nowUs, numFramesPlayed, 5315d313803bdd61d59f28e518e7a793f33832f1901Andy Hung (long long)numFramesPlayedAtUs); 532c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return durationUs; 533c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 534c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 535c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::onDrainVideo_l() { 536c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (!isPlaying()) { 537c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return; 538c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 539c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 540c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia while (!mBufferItems.empty()) { 54180243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia int64_t nowUs = ALooper::GetNowUs(); 542c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia BufferItem *bufferItem = &*mBufferItems.begin(); 543c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t itemMediaUs = bufferItem->mTimestamp / 1000; 544c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t itemRealUs = getRealTime(itemMediaUs, nowUs); 54580243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia 546a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar // adjust video frame PTS based on vsync 547a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar itemRealUs = mFrameScheduler->schedule(itemRealUs * 1000) / 1000; 54882a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar int64_t twoVsyncsUs = 2 * (mFrameScheduler->getVsyncPeriod() / 1000); 549a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar 550a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar // post 2 display refreshes before rendering is due 551a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (itemRealUs <= nowUs + twoVsyncsUs) { 552a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar ALOGV("adjusting PTS from %lld to %lld", 553a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar (long long)bufferItem->mTimestamp / 1000, (long long)itemRealUs); 554a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar bufferItem->mTimestamp = itemRealUs * 1000; 555a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar bufferItem->mIsAutoTimestamp = false; 556a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar 557c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mHasAudio) { 558c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) { 55982a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar renderOneBufferItem_l(*bufferItem); 560c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } else { 561c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // too late. 562c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia returnBufferToInput_l( 563c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem->mGraphicBuffer, bufferItem->mFence); 564a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mFrameScheduler->restart(); 565c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 566c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } else { 567c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // always render video buffer in video-only mode. 56882a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar renderOneBufferItem_l(*bufferItem); 569c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 570c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // smooth out videos >= 10fps 571c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mMediaClock->updateAnchor( 572c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia itemMediaUs, nowUs, itemMediaUs + 100000); 573c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 574c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 575c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mBufferItems.erase(mBufferItems.begin()); 57680243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia mNextBufferItemMediaUs = -1; 577c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } else { 578c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mNextBufferItemMediaUs == -1 57980243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia || mNextBufferItemMediaUs > itemMediaUs) { 580c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<AMessage> msg = new AMessage(kWhatDrainVideo, this); 581a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar msg->post(itemRealUs - nowUs - twoVsyncsUs); 58280243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia mNextBufferItemMediaUs = itemMediaUs; 583c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 584c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia break; 585c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 586c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 587c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 588c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 589c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::onFrameAvailableFromInput() { 590c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 591c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 592a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar const static nsecs_t kAcquireWaitTimeout = 2000000000; // 2 seconds 593a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar 59408dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mReturnPendingInputFrame = false; 59508dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar 596c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // If there are too many outstanding buffers, wait until a buffer is 597c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // released back to the input in onBufferReleased. 598a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar // NOTE: BufferQueue allows dequeuing maxAcquiredBufferCount + 1 buffers 59908dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar while (mNumOutstandingBuffers > mMaxAcquiredBufferCount 60008dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar && !mIsAbandoned && !mReturnPendingInputFrame) { 601a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (mReleaseCondition.waitRelative(mMutex, kAcquireWaitTimeout) != OK) { 60282a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar ALOGI_IF(mPlaybackRate != 0.f, "still waiting to release a buffer before acquire"); 603a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 604c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 605c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // If the sync is abandoned while we are waiting, the release 606c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // condition variable will be broadcast, and we should just return 607c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // without attempting to do anything more (since the input queue will 608c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // also be abandoned). 609c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mIsAbandoned) { 610c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return; 611c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 612c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 613c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 614c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // Acquire and detach the buffer from the input. 615c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia BufferItem bufferItem; 616c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status_t status = mInput->acquireBuffer(&bufferItem, 0 /* presentWhen */); 617c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status != NO_ERROR) { 618c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE("acquiring buffer from input failed (%d)", status); 619c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return; 620c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 621264bac95912efe121d6a60026612617f04f42966Lajos Molnar ++mNumOutstandingBuffers; 622c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 623c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGV("acquired buffer %#llx from input", (long long)bufferItem.mGraphicBuffer->getId()); 624c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 625a8f5e0c24143299e3b3d722487de7322f7761559Pablo Ceballos status = mInput->detachBuffer(bufferItem.mSlot); 626c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status != NO_ERROR) { 627c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE("detaching buffer from input failed (%d)", status); 628c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status == NO_INIT) { 629c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // If the input has been abandoned, move on. 630c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia onAbandoned_l(true /* isInput */); 631c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 632c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return; 633c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 634c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 635bac588208fadec78e3fa205c238089df277d3817Wei Jia if (mBuffersFromInput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) { 636bac588208fadec78e3fa205c238089df277d3817Wei Jia // Something is wrong since this buffer should be at our hands, bail. 637a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar ALOGE("received buffer multiple times from input"); 638bac588208fadec78e3fa205c238089df277d3817Wei Jia mInput->consumerDisconnect(); 639bac588208fadec78e3fa205c238089df277d3817Wei Jia onAbandoned_l(true /* isInput */); 640bac588208fadec78e3fa205c238089df277d3817Wei Jia return; 641bac588208fadec78e3fa205c238089df277d3817Wei Jia } 642bac588208fadec78e3fa205c238089df277d3817Wei Jia mBuffersFromInput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer); 643bac588208fadec78e3fa205c238089df277d3817Wei Jia 64408dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar // If flush happened while waiting for a buffer to be released, simply return it 64508dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar // TRICKY: do it here after it is detached so that we don't have to cache mGraphicBuffer. 64608dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar if (mReturnPendingInputFrame) { 64708dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mReturnPendingInputFrame = false; 64808dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence); 64908dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar return; 65008dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar } 65108dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar 652c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mBufferItems.push_back(bufferItem); 653bac588208fadec78e3fa205c238089df277d3817Wei Jia 65480243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia if (mBufferItems.size() == 1) { 65580243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia onDrainVideo_l(); 65680243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia } 657c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 658c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 65982a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnarvoid MediaSync::renderOneBufferItem_l(const BufferItem &bufferItem) { 660c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia IGraphicBufferProducer::QueueBufferInput queueInput( 661c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem.mTimestamp, 662c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem.mIsAutoTimestamp, 663c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem.mDataSpace, 664c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem.mCrop, 665c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia static_cast<int32_t>(bufferItem.mScalingMode), 666c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem.mTransform, 667c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem.mFence); 668c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 669c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // Attach and queue the buffer to the output. 670c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int slot; 671264bac95912efe121d6a60026612617f04f42966Lajos Molnar mOutput->setGenerationNumber(bufferItem.mGraphicBuffer->getGenerationNumber()); 672c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status_t status = mOutput->attachBuffer(&slot, bufferItem.mGraphicBuffer); 673c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE_IF(status != NO_ERROR, "attaching buffer to output failed (%d)", status); 674c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status == NO_ERROR) { 675c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia IGraphicBufferProducer::QueueBufferOutput queueOutput; 676c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status = mOutput->queueBuffer(slot, queueInput, &queueOutput); 677c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE_IF(status != NO_ERROR, "queueing buffer to output failed (%d)", status); 678c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 679c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 680c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status != NO_ERROR) { 681c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence); 682c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status == NO_INIT) { 683c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // If the output has been abandoned, move on. 684c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia onAbandoned_l(false /* isInput */); 685c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 686c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return; 687c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 688c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 689b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia if (mBuffersSentToOutput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) { 690b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia // Something is wrong since this buffer should be held by output now, bail. 691b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mInput->consumerDisconnect(); 692b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia onAbandoned_l(true /* isInput */); 693b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia return; 694b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia } 695b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mBuffersSentToOutput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer); 696b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia 697c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId()); 698c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 699c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 700b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jiavoid MediaSync::onBufferReleasedByOutput(sp<IGraphicBufferProducer> &output) { 701c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 702c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 703b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia if (output != mOutput) { 704b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia return; // This is not the current output, ignore. 705b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia } 706b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia 707c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<GraphicBuffer> buffer; 708c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<Fence> fence; 70982a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar status_t status = mOutput->detachNextBuffer(&buffer, &fence); 71082a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status); 711c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 71282a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar if (status == NO_INIT) { 71382a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar // If the output has been abandoned, we can't do anything else, 71482a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar // since buffer is invalid. 71582a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar onAbandoned_l(false /* isInput */); 71682a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar return; 71782a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar } 718c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 71982a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar ALOGV("detached buffer %#llx from output", (long long)buffer->getId()); 720c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 72182a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar // If we've been abandoned, we can't return the buffer to the input, so just 72282a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar // move on. 72382a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar if (mIsAbandoned) { 72482a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar return; 72582a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar } 726b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia 72782a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar ssize_t ix = mBuffersSentToOutput.indexOfKey(buffer->getId()); 72882a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar if (ix < 0) { 72982a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar // The buffer is unknown, maybe leftover, ignore. 73082a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar return; 731a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 73282a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar mBuffersSentToOutput.removeItemsAt(ix); 73382a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar 73482a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar returnBufferToInput_l(buffer, fence); 735c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 736c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 737c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::returnBufferToInput_l( 738c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia const sp<GraphicBuffer> &buffer, const sp<Fence> &fence) { 739bac588208fadec78e3fa205c238089df277d3817Wei Jia ssize_t ix = mBuffersFromInput.indexOfKey(buffer->getId()); 740bac588208fadec78e3fa205c238089df277d3817Wei Jia if (ix < 0) { 741bac588208fadec78e3fa205c238089df277d3817Wei Jia // The buffer is unknown, something is wrong, bail. 742a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar ALOGE("output returned unknown buffer"); 743bac588208fadec78e3fa205c238089df277d3817Wei Jia mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 744bac588208fadec78e3fa205c238089df277d3817Wei Jia onAbandoned_l(false /* isInput */); 745bac588208fadec78e3fa205c238089df277d3817Wei Jia return; 746bac588208fadec78e3fa205c238089df277d3817Wei Jia } 747bac588208fadec78e3fa205c238089df277d3817Wei Jia sp<GraphicBuffer> oldBuffer = mBuffersFromInput.valueAt(ix); 748bac588208fadec78e3fa205c238089df277d3817Wei Jia mBuffersFromInput.removeItemsAt(ix); 749bac588208fadec78e3fa205c238089df277d3817Wei Jia 750c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // Attach and release the buffer back to the input. 751c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int consumerSlot; 752bac588208fadec78e3fa205c238089df277d3817Wei Jia status_t status = mInput->attachBuffer(&consumerSlot, oldBuffer); 753c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE_IF(status != NO_ERROR, "attaching buffer to input failed (%d)", status); 754c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status == NO_ERROR) { 755c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status = mInput->releaseBuffer(consumerSlot, 0 /* frameNumber */, 756c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence); 757c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE_IF(status != NO_ERROR, "releasing buffer to input failed (%d)", status); 758c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 759c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 760c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // Notify any waiting onFrameAvailable calls. 761c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia --mNumOutstandingBuffers; 762c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mReleaseCondition.signal(); 763264bac95912efe121d6a60026612617f04f42966Lajos Molnar 764264bac95912efe121d6a60026612617f04f42966Lajos Molnar if (status == NO_ERROR) { 765264bac95912efe121d6a60026612617f04f42966Lajos Molnar ALOGV("released buffer %#llx to input", (long long)oldBuffer->getId()); 766264bac95912efe121d6a60026612617f04f42966Lajos Molnar } 767c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 768c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 769c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::onAbandoned_l(bool isInput) { 770c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE("the %s has abandoned me", (isInput ? "input" : "output")); 771c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (!mIsAbandoned) { 772c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (isInput) { 773c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 774c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } else { 775c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mInput->consumerDisconnect(); 776c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 777c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mIsAbandoned = true; 778c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 779c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mReleaseCondition.broadcast(); 780c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 781c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 782c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::onMessageReceived(const sp<AMessage> &msg) { 783c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia switch (msg->what()) { 784c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia case kWhatDrainVideo: 785c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia { 786c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 78780243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia if (mNextBufferItemMediaUs != -1) { 78880243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia int64_t nowUs = ALooper::GetNowUs(); 78980243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia int64_t itemRealUs = getRealTime(mNextBufferItemMediaUs, nowUs); 79080243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia 79180243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia // The message could arrive earlier than expected due to 79280243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia // various reasons, e.g., media clock has been changed because 79380243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia // of new anchor time or playback rate. In such cases, the 79480243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia // message needs to be re-posted. 79580243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia if (itemRealUs > nowUs) { 79680243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia msg->post(itemRealUs - nowUs); 79780243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia break; 79880243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia } 79980243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia } 80080243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia 801c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia onDrainVideo_l(); 802c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia break; 803c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 804c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 805c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia default: 806c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia TRESPASS(); 807c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia break; 808c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 809c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 810c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 811c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei JiaMediaSync::InputListener::InputListener(const sp<MediaSync> &sync) 812c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia : mSync(sync) {} 813c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 814c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei JiaMediaSync::InputListener::~InputListener() {} 815c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 816c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::InputListener::onFrameAvailable(const BufferItem &/* item */) { 817c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mSync->onFrameAvailableFromInput(); 818c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 819c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 820c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia// We don't care about sideband streams, since we won't relay them. 821c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::InputListener::onSidebandStreamChanged() { 822c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE("onSidebandStreamChanged: got sideband stream unexpectedly."); 823c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 824c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 825c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 826c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::InputListener::binderDied(const wp<IBinder> &/* who */) { 827c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mSync->mMutex); 828c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mSync->onAbandoned_l(true /* isInput */); 829c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 830c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 831b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei JiaMediaSync::OutputListener::OutputListener(const sp<MediaSync> &sync, 832b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia const sp<IGraphicBufferProducer> &output) 833b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia : mSync(sync), 834b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mOutput(output) {} 835c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 836c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei JiaMediaSync::OutputListener::~OutputListener() {} 837c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 838c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::OutputListener::onBufferReleased() { 839b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mSync->onBufferReleasedByOutput(mOutput); 840c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 841c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 842c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::OutputListener::binderDied(const wp<IBinder> &/* who */) { 843c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mSync->mMutex); 844c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mSync->onAbandoned_l(false /* isInput */); 845c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 846c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 847c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} // namespace android 848