1c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia/* 2c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * Copyright 2015 The Android Open Source Project 3c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * 4c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * Licensed under the Apache License, Version 2.0 (the "License"); 5c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * you may not use this file except in compliance with the License. 6c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * You may obtain a copy of the License at 7c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * 8c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * http://www.apache.org/licenses/LICENSE-2.0 9c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * 10c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * Unless required by applicable law or agreed to in writing, software 11c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * distributed under the License is distributed on an "AS IS" BASIS, 12c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * See the License for the specific language governing permissions and 14c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia * limitations under the License. 15c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia */ 16c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 17c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia//#define LOG_NDEBUG 0 18c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#define LOG_TAG "MediaSync" 19c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <inttypes.h> 20c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 21c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <gui/BufferQueue.h> 22c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <gui/IGraphicBufferConsumer.h> 23c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <gui/IGraphicBufferProducer.h> 24c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 25c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <media/AudioTrack.h> 26c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <media/stagefright/MediaClock.h> 27c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <media/stagefright/MediaSync.h> 28a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar#include <media/stagefright/VideoFrameScheduler.h> 29c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <media/stagefright/foundation/ADebug.h> 30c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <media/stagefright/foundation/ALooper.h> 31c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <media/stagefright/foundation/AMessage.h> 32c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 33c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia#include <ui/GraphicBuffer.h> 34c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 35defb1b0b7e082621a10763d1bd7a4a01e280fdf0Mathias Agopian#include <system/window.h> 36defb1b0b7e082621a10763d1bd7a4a01e280fdf0Mathias Agopian 37c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia// Maximum late time allowed for a video frame to be rendered. When a video 38c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia// frame arrives later than this number, it will be discarded without rendering. 39c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiastatic const int64_t kMaxAllowedVideoLateTimeUs = 40000ll; 40c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 41c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jianamespace android { 42c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 43c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia// static 44c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiasp<MediaSync> MediaSync::create() { 45c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<MediaSync> sync = new MediaSync(); 46c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sync->mLooper->registerHandler(sync); 47c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return sync; 48c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 49c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 50c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei JiaMediaSync::MediaSync() 51c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia : mIsAbandoned(false), 52c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mMutex(), 53c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mReleaseCondition(), 54c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mNumOutstandingBuffers(0), 55b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mUsageFlagsFromOutput(0), 56a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mMaxAcquiredBufferCount(1), 5708dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mReturnPendingInputFrame(false), 58c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mNativeSampleRateInHz(0), 59c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mNumFramesWritten(0), 60c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mHasAudio(false), 61c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mNextBufferItemMediaUs(-1), 62c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mPlaybackRate(0.0) { 63c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mMediaClock = new MediaClock; 64c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 653a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // initialize settings 663a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT; 673a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mPlaybackSettings.mSpeed = mPlaybackRate; 683a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 69c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mLooper = new ALooper; 70c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mLooper->setName("MediaSync"); 71c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 72c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 73c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 74c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei JiaMediaSync::~MediaSync() { 75c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mInput != NULL) { 76c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mInput->consumerDisconnect(); 77c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 78c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mOutput != NULL) { 79c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 80c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 81c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 82c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mLooper != NULL) { 83c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mLooper->unregisterHandler(id()); 84c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mLooper->stop(); 85c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 86c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 87c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 88ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jiastatus_t MediaSync::setSurface(const sp<IGraphicBufferProducer> &output) { 89c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 90c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 91b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia if (output == mOutput) { 92b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia return NO_ERROR; // same output surface. 93c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 94c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 953a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (output == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_VSYNC) { 96ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jia ALOGE("setSurface: output surface is used as sync source and cannot be removed."); 973a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return INVALID_OPERATION; 983a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 993a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 100c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (output != NULL) { 101b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia int newUsage = 0; 102b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia output->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &newUsage); 103b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia 104b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia // Check usage flags only when current output surface has been used to create input surface. 105b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia if (mOutput != NULL && mInput != NULL) { 106b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia int ignoredFlags = (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_COMPOSER 107b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia | GRALLOC_USAGE_EXTERNAL_DISP); 108b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia // New output surface is not allowed to add new usage flag except ignored ones. 109b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia if ((newUsage & ~(mUsageFlagsFromOutput | ignoredFlags)) != 0) { 110b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia ALOGE("setSurface: new output surface has new usage flag not used by current one."); 111b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia return BAD_VALUE; 112b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia } 113b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia } 114b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia 115b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia // Try to connect to new output surface. If failed, current output surface will not 116b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia // be changed. 117c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia IGraphicBufferProducer::QueueBufferOutput queueBufferOutput; 118b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia sp<OutputListener> listener(new OutputListener(this, output)); 119c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia IInterface::asBinder(output)->linkToDeath(listener); 120c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status_t status = 121c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia output->connect(listener, 122c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia NATIVE_WINDOW_API_MEDIA, 123c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia true /* producerControlledByApp */, 124c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia &queueBufferOutput); 125c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status != NO_ERROR) { 126ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jia ALOGE("setSurface: failed to connect (%d)", status); 127c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return status; 128c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 129a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar 130a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (mFrameScheduler == NULL) { 131a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mFrameScheduler = new VideoFrameScheduler(); 132a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mFrameScheduler->init(); 133a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 134b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia } 135c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 136b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia if (mOutput != NULL) { 137b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 138b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia while (!mBuffersSentToOutput.isEmpty()) { 139b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia returnBufferToInput_l(mBuffersSentToOutput.valueAt(0), Fence::NO_FENCE); 140b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mBuffersSentToOutput.removeItemsAt(0); 141b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia } 142c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 143c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 144b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mOutput = output; 145b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia 146c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return NO_ERROR; 147c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 148c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 149c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia// |audioTrack| is used only for querying information. 150ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jiastatus_t MediaSync::setAudioTrack(const sp<AudioTrack> &audioTrack) { 151c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 152c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 153c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // TODO: support audio track change. 154c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mAudioTrack != NULL) { 155ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jia ALOGE("setAudioTrack: audioTrack has already been configured."); 156c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return INVALID_OPERATION; 157c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 158c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 1593a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (audioTrack == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_AUDIO) { 160ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jia ALOGE("setAudioTrack: audioTrack is used as sync source and cannot be removed."); 1613a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return INVALID_OPERATION; 1629767a47dcf9bbe090045ab99e0290d73289c704fWei Jia } 1639767a47dcf9bbe090045ab99e0290d73289c704fWei Jia 1643a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (audioTrack != NULL) { 1653a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // check if audio track supports the playback settings 1663a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (mPlaybackSettings.mSpeed != 0.f 1673a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar && audioTrack->setPlaybackRate(mPlaybackSettings) != OK) { 1683a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar ALOGE("playback settings are not supported by the audio track"); 1693a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return INVALID_OPERATION; 1703a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 1713a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar uint32_t nativeSampleRateInHz = audioTrack->getOriginalSampleRate(); 1723a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (nativeSampleRateInHz <= 0) { 173ce8c218ec9db7acd86f9f8929f8bbb668151b53dWei Jia ALOGE("setAudioTrack: native sample rate should be positive."); 1743a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return BAD_VALUE; 1753a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 1763a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mAudioTrack = audioTrack; 1773a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mNativeSampleRateInHz = nativeSampleRateInHz; 1783a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar (void)setPlaybackSettings_l(mPlaybackSettings); 1793a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 1803a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar else { 1813a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mAudioTrack = NULL; 1823a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mNativeSampleRateInHz = 0; 1833a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 184c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 1853a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // potentially resync to new source 1863a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar resync_l(); 1873a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return OK; 188c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 189c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 190c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiastatus_t MediaSync::createInputSurface( 191c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<IGraphicBufferProducer> *outBufferProducer) { 192c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (outBufferProducer == NULL) { 193c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return BAD_VALUE; 194c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 195c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 196c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 197c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 198c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mOutput == NULL) { 199c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return NO_INIT; 200c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 201c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 202c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mInput != NULL) { 203c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return INVALID_OPERATION; 204c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 205c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 206c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<IGraphicBufferProducer> bufferProducer; 207c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<IGraphicBufferConsumer> bufferConsumer; 208c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia BufferQueue::createBufferQueue(&bufferProducer, &bufferConsumer); 209c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 210c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<InputListener> listener(new InputListener(this)); 211c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia IInterface::asBinder(bufferConsumer)->linkToDeath(listener); 212c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status_t status = 213c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferConsumer->consumerConnect(listener, false /* controlledByApp */); 214c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status == NO_ERROR) { 215c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferConsumer->setConsumerName(String8("MediaSync")); 2169b73fa9a02b7234b43ac2be66a3328ff36d39c2dLajos Molnar // propagate usage bits from output surface 217b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mUsageFlagsFromOutput = 0; 218b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mOutput->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &mUsageFlagsFromOutput); 219b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia bufferConsumer->setConsumerUsageBits(mUsageFlagsFromOutput); 220c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia *outBufferProducer = bufferProducer; 221c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mInput = bufferConsumer; 222a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar 223a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar // set undequeued buffer count 224a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar int minUndequeuedBuffers; 225a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mOutput->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers); 226a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mMaxAcquiredBufferCount = minUndequeuedBuffers; 227a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar bufferConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount); 228c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 229c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return status; 230c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 231c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 2323a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarvoid MediaSync::resync_l() { 2333a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar AVSyncSource src = mSyncSettings.mSource; 2343a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (src == AVSYNC_SOURCE_DEFAULT) { 2353a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (mAudioTrack != NULL) { 2363a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar src = AVSYNC_SOURCE_AUDIO; 2373a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } else { 2383a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar src = AVSYNC_SOURCE_SYSTEM_CLOCK; 2393a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 240c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 241c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 2423a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // TODO: resync ourselves to the current clock (e.g. on sync source change) 2433a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar updatePlaybackRate_l(mPlaybackRate); 2443a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 245c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 2463a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarvoid MediaSync::updatePlaybackRate_l(float rate) { 247c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (rate > mPlaybackRate) { 248c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mNextBufferItemMediaUs = -1; 249c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 250c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mPlaybackRate = rate; 25108dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar // TODO: update frame scheduler with this info 252c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mMediaClock->setPlaybackRate(rate); 253c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia onDrainVideo_l(); 254c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 255c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 256c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiasp<const MediaClock> MediaSync::getMediaClock() { 257c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return mMediaClock; 258c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 259c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 26099d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jiastatus_t MediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) { 26199d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia Mutex::Autolock lock(mMutex); 26299d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia // User should check the playback rate if it doesn't want to receive a 26399d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia // huge number for play time. 26499d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia if (mPlaybackRate == 0.0f) { 26599d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia *outTimeUs = INT64_MAX; 26699d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia return OK; 26799d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia } 26899d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia 26999d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia uint32_t numFramesPlayed = 0; 27099d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia if (mAudioTrack != NULL) { 27199d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia status_t res = mAudioTrack->getPosition(&numFramesPlayed); 27299d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia if (res != OK) { 27399d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia return res; 27499d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia } 27599d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia } 27699d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia 27799d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia int64_t numPendingFrames = mNumFramesWritten - numFramesPlayed; 27899d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia if (numPendingFrames < 0) { 27999d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia numPendingFrames = 0; 28099d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia ALOGW("getPlayTimeForPendingAudioFrames: pending frame count is negative."); 28199d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia } 28299d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia double timeUs = numPendingFrames * 1000000.0 28399d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia / (mNativeSampleRateInHz * (double)mPlaybackRate); 28499d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia if (timeUs > (double)INT64_MAX) { 28599d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia // Overflow. 28699d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia *outTimeUs = INT64_MAX; 28799d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia ALOGW("getPlayTimeForPendingAudioFrames: play time for pending audio frames " 28899d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia "is too high, possibly due to super low playback rate(%f)", mPlaybackRate); 28999d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia } else { 29099d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia *outTimeUs = (int64_t)timeUs; 29199d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia } 29299d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia 29399d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia return OK; 29499d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia} 29599d1f78c9b16b5668e78c353373e0e7f4592cab9Wei Jia 296c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiastatus_t MediaSync::updateQueuedAudioData( 297c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia size_t sizeInBytes, int64_t presentationTimeUs) { 298c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (sizeInBytes == 0) { 299c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return OK; 300c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 301c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 302c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 303c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 304c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mAudioTrack == NULL) { 305c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGW("updateQueuedAudioData: audioTrack has NOT been configured."); 306c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return INVALID_OPERATION; 307c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 308c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 309c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t numFrames = sizeInBytes / mAudioTrack->frameSize(); 310c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t maxMediaTimeUs = presentationTimeUs 311c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia + getDurationIfPlayedAtNativeSampleRate_l(numFrames); 312c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 313c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t nowUs = ALooper::GetNowUs(); 31480243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia int64_t nowMediaUs = presentationTimeUs 315c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia - getDurationIfPlayedAtNativeSampleRate_l(mNumFramesWritten) 316c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia + getPlayedOutAudioDurationMedia_l(nowUs); 317c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 31880243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia mNumFramesWritten += numFrames; 31980243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia 320c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t oldRealTime = -1; 321c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mNextBufferItemMediaUs != -1) { 322c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia oldRealTime = getRealTime(mNextBufferItemMediaUs, nowUs); 323c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 324c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 325c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mMediaClock->updateAnchor(nowMediaUs, nowUs, maxMediaTimeUs); 326c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mHasAudio = true; 327c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 328c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (oldRealTime != -1) { 329c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t newRealTime = getRealTime(mNextBufferItemMediaUs, nowUs); 33080243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia if (newRealTime >= oldRealTime) { 33180243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia return OK; 332c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 333c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 334c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 33580243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia mNextBufferItemMediaUs = -1; 33680243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia onDrainVideo_l(); 337c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return OK; 338c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 339c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 340c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::setName(const AString &name) { 341c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 342c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mInput->setConsumerName(String8(name.c_str())); 343c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 344c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 34508dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnarvoid MediaSync::flush() { 34608dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar Mutex::Autolock lock(mMutex); 34708dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar if (mFrameScheduler != NULL) { 34808dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mFrameScheduler->restart(); 34908dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar } 35008dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar while (!mBufferItems.empty()) { 35108dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar BufferItem *bufferItem = &*mBufferItems.begin(); 35208dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar returnBufferToInput_l(bufferItem->mGraphicBuffer, bufferItem->mFence); 35308dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mBufferItems.erase(mBufferItems.begin()); 35408dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar } 35508dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mNextBufferItemMediaUs = -1; 35608dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mNumFramesWritten = 0; 35708dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mReturnPendingInputFrame = true; 35808dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mReleaseCondition.signal(); 35908dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mMediaClock->clearAnchor(); 36008dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar} 36108dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar 3623a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarstatus_t MediaSync::setVideoFrameRateHint(float rate) { 363a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar Mutex::Autolock lock(mMutex); 364a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (rate < 0.f) { 365a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar return BAD_VALUE; 366a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 367a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (mFrameScheduler != NULL) { 368a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mFrameScheduler->init(rate); 369a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 370a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar return OK; 3713a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 3723a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 3733a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarfloat MediaSync::getVideoFrameRate() { 374a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar Mutex::Autolock lock(mMutex); 375a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (mFrameScheduler != NULL) { 376a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar float fps = mFrameScheduler->getFrameRate(); 377a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (fps > 0.f) { 378a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar return fps; 379a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 380a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 381a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar 382a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar // we don't have or know the frame rate 3833a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return -1.f; 3843a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 3853a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 3863a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarstatus_t MediaSync::setSyncSettings(const AVSyncSettings &syncSettings) { 3873a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // validate settings 3883a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (syncSettings.mSource >= AVSYNC_SOURCE_MAX 3893a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar || syncSettings.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX 3903a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar || syncSettings.mTolerance < 0.f 3913a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar || syncSettings.mTolerance >= AVSYNC_TOLERANCE_MAX) { 3923a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return BAD_VALUE; 3933a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 3943a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 3953a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar Mutex::Autolock lock(mMutex); 3963a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 3973a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // verify that we have the sync source 3983a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar switch (syncSettings.mSource) { 3993a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar case AVSYNC_SOURCE_AUDIO: 4003a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (mAudioTrack == NULL) { 4013a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar ALOGE("setSyncSettings: audio sync source requires an audio track"); 4023a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return BAD_VALUE; 4033a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4043a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar break; 4053a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar case AVSYNC_SOURCE_VSYNC: 4063a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (mOutput == NULL) { 4073a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar ALOGE("setSyncSettings: vsync sync source requires an output surface"); 4083a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return BAD_VALUE; 4093a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4103a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar break; 4113a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar default: 4123a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar break; 4133a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4143a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4153a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mSyncSettings = syncSettings; 4163a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar resync_l(); 4173a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return OK; 4183a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 4193a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4203a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarvoid MediaSync::getSyncSettings(AVSyncSettings *syncSettings) { 4213a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar Mutex::Autolock lock(mMutex); 4223a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar *syncSettings = mSyncSettings; 4233a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 4243a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4253a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarstatus_t MediaSync::setPlaybackSettings(const AudioPlaybackRate &rate) { 4263a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar Mutex::Autolock lock(mMutex); 4273a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4283a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar status_t err = setPlaybackSettings_l(rate); 4293a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (err == OK) { 4303a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // TODO: adjust rate if using VSYNC as source 4313a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar updatePlaybackRate_l(rate.mSpeed); 4323a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4333a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return err; 4343a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 4353a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4363a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarstatus_t MediaSync::setPlaybackSettings_l(const AudioPlaybackRate &rate) { 4373a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (rate.mSpeed < 0.f || rate.mPitch < 0.f) { 4383a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // We don't validate other audio settings. 4393a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // They will be validated when/if audiotrack is set. 4403a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return BAD_VALUE; 4413a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4423a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4433a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (mAudioTrack != NULL) { 4443a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (rate.mSpeed == 0.f) { 4453a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mAudioTrack->pause(); 4463a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } else { 4473a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar status_t err = mAudioTrack->setPlaybackRate(rate); 4483a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar if (err != OK) { 4493a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return BAD_VALUE; 4503a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4513a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4523a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar // ignore errors 4533a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar (void)mAudioTrack->start(); 4543a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4553a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar } 4563a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar mPlaybackSettings = rate; 4573a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar return OK; 4583a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 4593a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 4603a474aa67fc31505740526dd249d96204c08bf79Lajos Molnarvoid MediaSync::getPlaybackSettings(AudioPlaybackRate *rate) { 4613a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar Mutex::Autolock lock(mMutex); 4623a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar *rate = mPlaybackSettings; 4633a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar} 4643a474aa67fc31505740526dd249d96204c08bf79Lajos Molnar 465c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiaint64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) { 466c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t realUs; 467c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) { 468c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // If failed to get current position, e.g. due to audio clock is 469c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // not ready, then just play out video immediately without delay. 470c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return nowUs; 471c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 472c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return realUs; 473c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 474c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 475c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiaint64_t MediaSync::getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames) { 476c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return (numFrames * 1000000LL / mNativeSampleRateInHz); 477c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 478c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 479c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiaint64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) { 480c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia CHECK(mAudioTrack != NULL); 481c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 482c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia uint32_t numFramesPlayed; 4835d313803bdd61d59f28e518e7a793f33832f1901Andy Hung int64_t numFramesPlayedAtUs; 484c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia AudioTimestamp ts; 485c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 486c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status_t res = mAudioTrack->getTimestamp(ts); 487c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (res == OK) { 488c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // case 1: mixing audio tracks. 489c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia numFramesPlayed = ts.mPosition; 4905d313803bdd61d59f28e518e7a793f33832f1901Andy Hung numFramesPlayedAtUs = ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000; 491c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia //ALOGD("getTimestamp: OK %d %lld", 4925d313803bdd61d59f28e518e7a793f33832f1901Andy Hung // numFramesPlayed, (long long)numFramesPlayedAtUs); 493c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } else if (res == WOULD_BLOCK) { 494c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // case 2: transitory state on start of a new track 495c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia numFramesPlayed = 0; 4965d313803bdd61d59f28e518e7a793f33832f1901Andy Hung numFramesPlayedAtUs = nowUs; 497c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia //ALOGD("getTimestamp: WOULD_BLOCK %d %lld", 4985d313803bdd61d59f28e518e7a793f33832f1901Andy Hung // numFramesPlayed, (long long)numFramesPlayedAtUs); 499c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } else { 500c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // case 3: transitory at new track or audio fast tracks. 501c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia res = mAudioTrack->getPosition(&numFramesPlayed); 502c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia CHECK_EQ(res, (status_t)OK); 5035d313803bdd61d59f28e518e7a793f33832f1901Andy Hung numFramesPlayedAtUs = nowUs; 5045d313803bdd61d59f28e518e7a793f33832f1901Andy Hung numFramesPlayedAtUs += 1000LL * mAudioTrack->latency() / 2; /* XXX */ 5055d313803bdd61d59f28e518e7a793f33832f1901Andy Hung //ALOGD("getPosition: %d %lld", numFramesPlayed, (long long)numFramesPlayedAtUs); 506c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 507c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 508c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia //can't be negative until 12.4 hrs, test. 509c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia //CHECK_EQ(numFramesPlayed & (1 << 31), 0); 510c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t durationUs = 511c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia getDurationIfPlayedAtNativeSampleRate_l(numFramesPlayed) 5125d313803bdd61d59f28e518e7a793f33832f1901Andy Hung + nowUs - numFramesPlayedAtUs; 513c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (durationUs < 0) { 514c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // Occurs when numFramesPlayed position is very small and the following: 515c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // (1) In case 1, the time nowUs is computed before getTimestamp() is 5165d313803bdd61d59f28e518e7a793f33832f1901Andy Hung // called and numFramesPlayedAtUs is greater than nowUs by time more 517c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // than numFramesPlayed. 518c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // (2) In case 3, using getPosition and adding mAudioTrack->latency() 5195d313803bdd61d59f28e518e7a793f33832f1901Andy Hung // to numFramesPlayedAtUs, by a time amount greater than 520c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // numFramesPlayed. 521c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // 522c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // Both of these are transitory conditions. 523c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGV("getPlayedOutAudioDurationMedia_l: negative duration %lld " 524c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia "set to zero", (long long)durationUs); 525c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia durationUs = 0; 526c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 527c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGV("getPlayedOutAudioDurationMedia_l(%lld) nowUs(%lld) frames(%u) " 528c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia "framesAt(%lld)", 529c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia (long long)durationUs, (long long)nowUs, numFramesPlayed, 5305d313803bdd61d59f28e518e7a793f33832f1901Andy Hung (long long)numFramesPlayedAtUs); 531c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return durationUs; 532c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 533c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 534c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::onDrainVideo_l() { 535c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (!isPlaying()) { 536c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return; 537c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 538c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 539c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia while (!mBufferItems.empty()) { 54080243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia int64_t nowUs = ALooper::GetNowUs(); 541c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia BufferItem *bufferItem = &*mBufferItems.begin(); 542c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t itemMediaUs = bufferItem->mTimestamp / 1000; 543c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int64_t itemRealUs = getRealTime(itemMediaUs, nowUs); 54480243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia 545a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar // adjust video frame PTS based on vsync 546a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar itemRealUs = mFrameScheduler->schedule(itemRealUs * 1000) / 1000; 54782a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar int64_t twoVsyncsUs = 2 * (mFrameScheduler->getVsyncPeriod() / 1000); 548a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar 549a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar // post 2 display refreshes before rendering is due 550a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (itemRealUs <= nowUs + twoVsyncsUs) { 551a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar ALOGV("adjusting PTS from %lld to %lld", 552a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar (long long)bufferItem->mTimestamp / 1000, (long long)itemRealUs); 553a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar bufferItem->mTimestamp = itemRealUs * 1000; 554a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar bufferItem->mIsAutoTimestamp = false; 555a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar 556c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mHasAudio) { 557c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) { 55882a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar renderOneBufferItem_l(*bufferItem); 559c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } else { 560c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // too late. 561c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia returnBufferToInput_l( 562c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem->mGraphicBuffer, bufferItem->mFence); 563a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar mFrameScheduler->restart(); 564c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 565c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } else { 566c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // always render video buffer in video-only mode. 56782a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar renderOneBufferItem_l(*bufferItem); 568c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 569c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // smooth out videos >= 10fps 570c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mMediaClock->updateAnchor( 571c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia itemMediaUs, nowUs, itemMediaUs + 100000); 572c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 573c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 574c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mBufferItems.erase(mBufferItems.begin()); 57580243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia mNextBufferItemMediaUs = -1; 576c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } else { 577c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mNextBufferItemMediaUs == -1 57880243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia || mNextBufferItemMediaUs > itemMediaUs) { 579c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<AMessage> msg = new AMessage(kWhatDrainVideo, this); 580a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar msg->post(itemRealUs - nowUs - twoVsyncsUs); 58180243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia mNextBufferItemMediaUs = itemMediaUs; 582c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 583c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia break; 584c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 585c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 586c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 587c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 588c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::onFrameAvailableFromInput() { 589c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 590c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 591a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar const static nsecs_t kAcquireWaitTimeout = 2000000000; // 2 seconds 592a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar 59308dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mReturnPendingInputFrame = false; 59408dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar 595c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // If there are too many outstanding buffers, wait until a buffer is 596c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // released back to the input in onBufferReleased. 597a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar // NOTE: BufferQueue allows dequeuing maxAcquiredBufferCount + 1 buffers 59808dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar while (mNumOutstandingBuffers > mMaxAcquiredBufferCount 59908dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar && !mIsAbandoned && !mReturnPendingInputFrame) { 600a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar if (mReleaseCondition.waitRelative(mMutex, kAcquireWaitTimeout) != OK) { 60182a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar ALOGI_IF(mPlaybackRate != 0.f, "still waiting to release a buffer before acquire"); 602a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 603c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 604c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // If the sync is abandoned while we are waiting, the release 605c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // condition variable will be broadcast, and we should just return 606c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // without attempting to do anything more (since the input queue will 607c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // also be abandoned). 608c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (mIsAbandoned) { 609c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return; 610c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 611c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 612c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 613c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // Acquire and detach the buffer from the input. 614c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia BufferItem bufferItem; 615c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status_t status = mInput->acquireBuffer(&bufferItem, 0 /* presentWhen */); 616c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status != NO_ERROR) { 617c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE("acquiring buffer from input failed (%d)", status); 618c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return; 619c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 620264bac95912efe121d6a60026612617f04f42966Lajos Molnar ++mNumOutstandingBuffers; 621c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 622c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGV("acquired buffer %#llx from input", (long long)bufferItem.mGraphicBuffer->getId()); 623c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 624a8f5e0c24143299e3b3d722487de7322f7761559Pablo Ceballos status = mInput->detachBuffer(bufferItem.mSlot); 625c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status != NO_ERROR) { 626c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE("detaching buffer from input failed (%d)", status); 627c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status == NO_INIT) { 628c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // If the input has been abandoned, move on. 629c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia onAbandoned_l(true /* isInput */); 630c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 631c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return; 632c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 633c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 634bac588208fadec78e3fa205c238089df277d3817Wei Jia if (mBuffersFromInput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) { 635bac588208fadec78e3fa205c238089df277d3817Wei Jia // Something is wrong since this buffer should be at our hands, bail. 636a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar ALOGE("received buffer multiple times from input"); 637bac588208fadec78e3fa205c238089df277d3817Wei Jia mInput->consumerDisconnect(); 638bac588208fadec78e3fa205c238089df277d3817Wei Jia onAbandoned_l(true /* isInput */); 639bac588208fadec78e3fa205c238089df277d3817Wei Jia return; 640bac588208fadec78e3fa205c238089df277d3817Wei Jia } 641bac588208fadec78e3fa205c238089df277d3817Wei Jia mBuffersFromInput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer); 642bac588208fadec78e3fa205c238089df277d3817Wei Jia 64308dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar // If flush happened while waiting for a buffer to be released, simply return it 64408dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar // TRICKY: do it here after it is detached so that we don't have to cache mGraphicBuffer. 64508dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar if (mReturnPendingInputFrame) { 64608dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar mReturnPendingInputFrame = false; 64708dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence); 64808dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar return; 64908dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar } 65008dc42c46c942ff316a69be1cf74f3c60cf53e7fLajos Molnar 651c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mBufferItems.push_back(bufferItem); 652bac588208fadec78e3fa205c238089df277d3817Wei Jia 65380243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia if (mBufferItems.size() == 1) { 65480243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia onDrainVideo_l(); 65580243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia } 656c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 657c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 65882a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnarvoid MediaSync::renderOneBufferItem_l(const BufferItem &bufferItem) { 659c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia IGraphicBufferProducer::QueueBufferInput queueInput( 660c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem.mTimestamp, 661c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem.mIsAutoTimestamp, 662c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem.mDataSpace, 663c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem.mCrop, 664c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia static_cast<int32_t>(bufferItem.mScalingMode), 665c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem.mTransform, 666c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia bufferItem.mFence); 667c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 668c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // Attach and queue the buffer to the output. 669c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int slot; 670264bac95912efe121d6a60026612617f04f42966Lajos Molnar mOutput->setGenerationNumber(bufferItem.mGraphicBuffer->getGenerationNumber()); 671c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status_t status = mOutput->attachBuffer(&slot, bufferItem.mGraphicBuffer); 672c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE_IF(status != NO_ERROR, "attaching buffer to output failed (%d)", status); 673c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status == NO_ERROR) { 674c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia IGraphicBufferProducer::QueueBufferOutput queueOutput; 675c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status = mOutput->queueBuffer(slot, queueInput, &queueOutput); 676c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE_IF(status != NO_ERROR, "queueing buffer to output failed (%d)", status); 677c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 678c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 679c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status != NO_ERROR) { 680c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence); 681c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status == NO_INIT) { 682c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // If the output has been abandoned, move on. 683c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia onAbandoned_l(false /* isInput */); 684c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 685c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia return; 686c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 687c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 688b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia if (mBuffersSentToOutput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) { 689b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia // Something is wrong since this buffer should be held by output now, bail. 690b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mInput->consumerDisconnect(); 691b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia onAbandoned_l(true /* isInput */); 692b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia return; 693b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia } 694b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mBuffersSentToOutput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer); 695b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia 696c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId()); 697c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 698c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 699b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jiavoid MediaSync::onBufferReleasedByOutput(sp<IGraphicBufferProducer> &output) { 700c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 701c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 702b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia if (output != mOutput) { 703b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia return; // This is not the current output, ignore. 704b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia } 705b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia 706c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<GraphicBuffer> buffer; 707c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia sp<Fence> fence; 70882a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar status_t status = mOutput->detachNextBuffer(&buffer, &fence); 70982a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status); 710c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 71182a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar if (status == NO_INIT) { 71282a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar // If the output has been abandoned, we can't do anything else, 71382a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar // since buffer is invalid. 71482a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar onAbandoned_l(false /* isInput */); 71582a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar return; 71682a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar } 717c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 71882a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar ALOGV("detached buffer %#llx from output", (long long)buffer->getId()); 719c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 72082a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar // If we've been abandoned, we can't return the buffer to the input, so just 72182a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar // move on. 72282a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar if (mIsAbandoned) { 72382a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar return; 72482a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar } 725b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia 72682a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar ssize_t ix = mBuffersSentToOutput.indexOfKey(buffer->getId()); 72782a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar if (ix < 0) { 72882a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar // The buffer is unknown, maybe leftover, ignore. 72982a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar return; 730a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar } 73182a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar mBuffersSentToOutput.removeItemsAt(ix); 73282a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar 73382a633b602a7ffe1efd9886744393f52a093a2f3Lajos Molnar returnBufferToInput_l(buffer, fence); 734c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 735c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 736c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::returnBufferToInput_l( 737c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia const sp<GraphicBuffer> &buffer, const sp<Fence> &fence) { 738bac588208fadec78e3fa205c238089df277d3817Wei Jia ssize_t ix = mBuffersFromInput.indexOfKey(buffer->getId()); 739bac588208fadec78e3fa205c238089df277d3817Wei Jia if (ix < 0) { 740bac588208fadec78e3fa205c238089df277d3817Wei Jia // The buffer is unknown, something is wrong, bail. 741a3725d7b0cb79ddb49f81cba00a0164d8e645acdLajos Molnar ALOGE("output returned unknown buffer"); 742bac588208fadec78e3fa205c238089df277d3817Wei Jia mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 743bac588208fadec78e3fa205c238089df277d3817Wei Jia onAbandoned_l(false /* isInput */); 744bac588208fadec78e3fa205c238089df277d3817Wei Jia return; 745bac588208fadec78e3fa205c238089df277d3817Wei Jia } 746bac588208fadec78e3fa205c238089df277d3817Wei Jia sp<GraphicBuffer> oldBuffer = mBuffersFromInput.valueAt(ix); 747bac588208fadec78e3fa205c238089df277d3817Wei Jia mBuffersFromInput.removeItemsAt(ix); 748bac588208fadec78e3fa205c238089df277d3817Wei Jia 749c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // Attach and release the buffer back to the input. 750c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia int consumerSlot; 751bac588208fadec78e3fa205c238089df277d3817Wei Jia status_t status = mInput->attachBuffer(&consumerSlot, oldBuffer); 752c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE_IF(status != NO_ERROR, "attaching buffer to input failed (%d)", status); 753c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (status == NO_ERROR) { 754c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia status = mInput->releaseBuffer(consumerSlot, 0 /* frameNumber */, 755c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence); 756c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE_IF(status != NO_ERROR, "releasing buffer to input failed (%d)", status); 757c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 758c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 759c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia // Notify any waiting onFrameAvailable calls. 760c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia --mNumOutstandingBuffers; 761c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mReleaseCondition.signal(); 762264bac95912efe121d6a60026612617f04f42966Lajos Molnar 763264bac95912efe121d6a60026612617f04f42966Lajos Molnar if (status == NO_ERROR) { 764264bac95912efe121d6a60026612617f04f42966Lajos Molnar ALOGV("released buffer %#llx to input", (long long)oldBuffer->getId()); 765264bac95912efe121d6a60026612617f04f42966Lajos Molnar } 766c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 767c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 768c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::onAbandoned_l(bool isInput) { 769c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE("the %s has abandoned me", (isInput ? "input" : "output")); 770c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (!mIsAbandoned) { 771c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia if (isInput) { 772c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 773c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } else { 774c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mInput->consumerDisconnect(); 775c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 776c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mIsAbandoned = true; 777c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 778c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mReleaseCondition.broadcast(); 779c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 780c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 781c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::onMessageReceived(const sp<AMessage> &msg) { 782c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia switch (msg->what()) { 783c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia case kWhatDrainVideo: 784c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia { 785c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mMutex); 78680243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia if (mNextBufferItemMediaUs != -1) { 78780243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia int64_t nowUs = ALooper::GetNowUs(); 78880243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia int64_t itemRealUs = getRealTime(mNextBufferItemMediaUs, nowUs); 78980243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia 79080243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia // The message could arrive earlier than expected due to 79180243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia // various reasons, e.g., media clock has been changed because 79280243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia // of new anchor time or playback rate. In such cases, the 79380243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia // message needs to be re-posted. 79480243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia if (itemRealUs > nowUs) { 79580243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia msg->post(itemRealUs - nowUs); 79680243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia break; 79780243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia } 79880243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia } 79980243639b0aa2c7c941c23c1aaeaa75de38c148cWei Jia 800c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia onDrainVideo_l(); 801c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia break; 802c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 803c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 804c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia default: 805c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia TRESPASS(); 806c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia break; 807c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia } 808c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 809c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 810c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei JiaMediaSync::InputListener::InputListener(const sp<MediaSync> &sync) 811c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia : mSync(sync) {} 812c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 813c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei JiaMediaSync::InputListener::~InputListener() {} 814c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 815c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::InputListener::onFrameAvailable(const BufferItem &/* item */) { 816c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mSync->onFrameAvailableFromInput(); 817c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 818c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 819c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia// We don't care about sideband streams, since we won't relay them. 820c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::InputListener::onSidebandStreamChanged() { 821c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia ALOGE("onSidebandStreamChanged: got sideband stream unexpectedly."); 822c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 823c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 824c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 825c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::InputListener::binderDied(const wp<IBinder> &/* who */) { 826c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mSync->mMutex); 827c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mSync->onAbandoned_l(true /* isInput */); 828c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 829c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 830b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei JiaMediaSync::OutputListener::OutputListener(const sp<MediaSync> &sync, 831b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia const sp<IGraphicBufferProducer> &output) 832b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia : mSync(sync), 833b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mOutput(output) {} 834c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 835c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei JiaMediaSync::OutputListener::~OutputListener() {} 836c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 837c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::OutputListener::onBufferReleased() { 838b6ea1292f8adae45d95c6f37d50c96534219b6d2Wei Jia mSync->onBufferReleasedByOutput(mOutput); 839c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 840c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 841c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jiavoid MediaSync::OutputListener::binderDied(const wp<IBinder> &/* who */) { 842c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia Mutex::Autolock lock(mSync->mMutex); 843c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia mSync->onAbandoned_l(false /* isInput */); 844c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} 845c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia 846c8db9712d9abe9b0d74193ea7d7cff428e32e62cWei Jia} // namespace android 847