1511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall/* 2511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall * 4511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall * Use of this source code is governed by a BSD-style license 5511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall * that can be found in the LICENSE file in the root of the source 6511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall * tree. An additional intellectual property rights grant can be found 7511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall * in the file PATENTS. All contributing project authors may 8511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall * be found in the AUTHORS file in the root of the source tree. 9511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall */ 10511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 11511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall#include "webrtc/modules/video_render/video_render_frames.h" 12511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 13511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall#include <assert.h> 14511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 15511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall#include "webrtc/common_video/interface/texture_video_frame.h" 16511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall#include "webrtc/modules/interface/module_common_types.h" 17511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall#include "webrtc/system_wrappers/interface/tick_util.h" 18511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall#include "webrtc/system_wrappers/interface/trace.h" 19511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 20511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrallnamespace webrtc { 21511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 22511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrallconst uint32_t KEventMaxWaitTimeMs = 200; 23511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrallconst uint32_t kMinRenderDelayMs = 10; 24511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrallconst uint32_t kMaxRenderDelayMs= 500; 25511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 26511eca30a483e912c274e1d8ba3a0f8f081e2227JP AbgrallVideoRenderFrames::VideoRenderFrames() 27511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall : render_delay_ms_(10) { 28511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall} 29511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 30511eca30a483e912c274e1d8ba3a0f8f081e2227JP AbgrallVideoRenderFrames::~VideoRenderFrames() { 31511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall ReleaseAllFrames(); 32511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall} 33511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 34511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrallint32_t VideoRenderFrames::AddFrame(I420VideoFrame* new_frame) { 35511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall const int64_t time_now = TickTime::MillisecondTimestamp(); 36511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 37511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall // Drop old frames only when there are other frames in the queue, otherwise, a 38511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall // really slow system never renders any frames. 39511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall if (!incoming_frames_.empty() && 40511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall new_frame->render_time_ms() + KOldRenderTimestampMS < time_now) { 41511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall WEBRTC_TRACE(kTraceWarning, 42511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall kTraceVideoRenderer, 43511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall -1, 44511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall "%s: too old frame, timestamp=%u.", 45511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall __FUNCTION__, 46511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall new_frame->timestamp()); 47511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall return -1; 48511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 49511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 50511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall if (new_frame->render_time_ms() > time_now + KFutureRenderTimestampMS) { 51511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall WEBRTC_TRACE(kTraceWarning, kTraceVideoRenderer, -1, 52511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall "%s: frame too long into the future, timestamp=%u.", 53511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall __FUNCTION__, new_frame->timestamp()); 54511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall return -1; 55511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 56511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 57511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall if (new_frame->native_handle() != NULL) { 58511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall incoming_frames_.push_back(new_frame->CloneFrame()); 59511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall return static_cast<int32_t>(incoming_frames_.size()); 60511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 61511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 62511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall // Get an empty frame 63511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall I420VideoFrame* frame_to_add = NULL; 64511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall if (!empty_frames_.empty()) { 65511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall frame_to_add = empty_frames_.front(); 66511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall empty_frames_.pop_front(); 67511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 68511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall if (!frame_to_add) { 69511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall if (empty_frames_.size() + incoming_frames_.size() > 70511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall KMaxNumberOfFrames) { 71511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall // Already allocated too many frames. 72511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall WEBRTC_TRACE(kTraceWarning, kTraceVideoRenderer, 73511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall -1, "%s: too many frames, timestamp=%u, limit=%d", 74511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall __FUNCTION__, new_frame->timestamp(), KMaxNumberOfFrames); 75511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall return -1; 76511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 77511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 78511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall // Allocate new memory. 79511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall WEBRTC_TRACE(kTraceMemory, kTraceVideoRenderer, -1, 80511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall "%s: allocating buffer %d", __FUNCTION__, 81511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall empty_frames_.size() + incoming_frames_.size()); 82511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 83511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall frame_to_add = new I420VideoFrame(); 84511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall if (!frame_to_add) { 85511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall WEBRTC_TRACE(kTraceError, kTraceVideoRenderer, -1, 86511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall "%s: could not create new frame for", __FUNCTION__); 87511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall return -1; 88511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 89511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 90511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 91511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall frame_to_add->CreateEmptyFrame(new_frame->width(), new_frame->height(), 92511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall new_frame->stride(kYPlane), 93511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall new_frame->stride(kUPlane), 94511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall new_frame->stride(kVPlane)); 95511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall // TODO(mflodman) Change this! 96511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall // Remove const ness. Copying will be costly. 97511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall frame_to_add->SwapFrame(new_frame); 98511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall incoming_frames_.push_back(frame_to_add); 99511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 100511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall return static_cast<int32_t>(incoming_frames_.size()); 101511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall} 102511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 103511eca30a483e912c274e1d8ba3a0f8f081e2227JP AbgrallI420VideoFrame* VideoRenderFrames::FrameToRender() { 104511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall I420VideoFrame* render_frame = NULL; 105511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall FrameList::iterator iter = incoming_frames_.begin(); 106511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall while(iter != incoming_frames_.end()) { 107511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall I420VideoFrame* oldest_frame_in_list = *iter; 108511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall if (oldest_frame_in_list->render_time_ms() <= 109511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall TickTime::MillisecondTimestamp() + render_delay_ms_) { 110511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall // This is the oldest one so far and it's OK to render. 111511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall if (render_frame) { 112511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall // This one is older than the newly found frame, remove this one. 113511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall ReturnFrame(render_frame); 114511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 115511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall render_frame = oldest_frame_in_list; 116511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall iter = incoming_frames_.erase(iter); 117511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } else { 118511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall // We can't release this one yet, we're done here. 119511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall break; 120511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 121511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 122511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall return render_frame; 123511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall} 124511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 125511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrallint32_t VideoRenderFrames::ReturnFrame(I420VideoFrame* old_frame) { 126511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall // No need to reuse texture frames because they do not allocate memory. 127511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall if (old_frame->native_handle() == NULL) { 128511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall old_frame->ResetSize(); 129511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall old_frame->set_timestamp(0); 130511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall old_frame->set_render_time_ms(0); 131511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall empty_frames_.push_back(old_frame); 132511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } else { 133511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall delete old_frame; 134511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 135511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall return 0; 136511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall} 137511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 138511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrallint32_t VideoRenderFrames::ReleaseAllFrames() { 139511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall for (FrameList::iterator iter = incoming_frames_.begin(); 140511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall iter != incoming_frames_.end(); ++iter) { 141511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall delete *iter; 142511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 143511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall incoming_frames_.clear(); 144511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 145511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall for (FrameList::iterator iter = empty_frames_.begin(); 146511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall iter != empty_frames_.end(); ++iter) { 147511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall delete *iter; 148511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 149511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall empty_frames_.clear(); 150511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall return 0; 151511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall} 152511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 153511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgralluint32_t VideoRenderFrames::TimeToNextFrameRelease() { 154511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall if (incoming_frames_.empty()) { 155511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall return KEventMaxWaitTimeMs; 156511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 157511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall I420VideoFrame* oldest_frame = incoming_frames_.front(); 158511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall int64_t time_to_release = oldest_frame->render_time_ms() - render_delay_ms_ 159511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall - TickTime::MillisecondTimestamp(); 160511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall if (time_to_release < 0) { 161511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall time_to_release = 0; 162511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 163511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall return static_cast<uint32_t>(time_to_release); 164511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall} 165511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 166511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrallint32_t VideoRenderFrames::SetRenderDelay( 167511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall const uint32_t render_delay) { 168511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall if (render_delay < kMinRenderDelayMs || 169511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall render_delay > kMaxRenderDelayMs) { 170511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall WEBRTC_TRACE(kTraceWarning, kTraceVideoRenderer, 171511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall -1, "%s(%d): Invalid argument.", __FUNCTION__, 172511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall render_delay); 173511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall return -1; 174511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall } 175511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 176511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall render_delay_ms_ = render_delay; 177511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall return 0; 178511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall} 179511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall 180511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall} // namespace webrtc 181511eca30a483e912c274e1d8ba3a0f8f081e2227JP Abgrall