1b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org/* 2b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * 4b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * Use of this source code is governed by a BSD-style license 5b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * that can be found in the LICENSE file in the root of the source 6b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * tree. An additional intellectual property rights grant can be found 7b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * in the file PATENTS. All contributing project authors may 8b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * be found in the AUTHORS file in the root of the source tree. 9b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org */ 10b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 11e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org#include "webrtc/modules/video_coding/main/source/receiver.h" 12b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 13b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org#include <assert.h> 14b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 15fee2074c4e0fcc65bef99ae4b7d9ae64e196276cpbos@webrtc.org#include <cstdlib> 16fee2074c4e0fcc65bef99ae4b7d9ae64e196276cpbos@webrtc.org 17e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org#include "webrtc/modules/video_coding/main/source/encoded_frame.h" 18e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org#include "webrtc/modules/video_coding/main/source/internal_defines.h" 19e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org#include "webrtc/modules/video_coding/main/source/media_opt_util.h" 201bb2146351979b6610107419b2a9c86cca2692a3stefan@webrtc.org#include "webrtc/system_wrappers/interface/clock.h" 218edcccef11eabb0dd98a061c9d640b2b7743609estefan@webrtc.org#include "webrtc/system_wrappers/interface/logging.h" 2274472fe9852769d834e135bde1229c24ab844244hclam@chromium.org#include "webrtc/system_wrappers/interface/trace_event.h" 23b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 24b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgnamespace webrtc { 25b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 269d6fcb37be4c9f49e1bb0b32bb01314c31f5e1ddmikhal@webrtc.orgenum { kMaxReceiverDelayMs = 10000 }; 279d6fcb37be4c9f49e1bb0b32bb01314c31f5e1ddmikhal@webrtc.org 28e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgVCMReceiver::VCMReceiver(VCMTiming* timing, 291bb2146351979b6610107419b2a9c86cca2692a3stefan@webrtc.org Clock* clock, 302637d61b86b68c7082ae21b4cc9bc88cc36d1c13stefan@webrtc.org EventFactory* event_factory, 31b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org bool master) 32e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), 33e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org clock_(clock), 34e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org master_(master), 358edcccef11eabb0dd98a061c9d640b2b7743609estefan@webrtc.org jitter_buffer_(clock_, event_factory), 36e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org timing_(timing), 372637d61b86b68c7082ae21b4cc9bc88cc36d1c13stefan@webrtc.org render_wait_event_(event_factory->CreateEvent()), 389d6fcb37be4c9f49e1bb0b32bb01314c31f5e1ddmikhal@webrtc.org state_(kPassive), 399d6fcb37be4c9f49e1bb0b32bb01314c31f5e1ddmikhal@webrtc.org max_video_delay_ms_(kMaxVideoDelayMs) {} 40e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org 41e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgVCMReceiver::~VCMReceiver() { 422637d61b86b68c7082ae21b4cc9bc88cc36d1c13stefan@webrtc.org render_wait_event_->Set(); 43e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org delete crit_sect_; 44b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 45b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 46e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgvoid VCMReceiver::Reset() { 47e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org CriticalSectionScoped cs(crit_sect_); 48e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org if (!jitter_buffer_.Running()) { 49e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org jitter_buffer_.Start(); 50e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } else { 51e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org jitter_buffer_.Flush(); 52e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 532637d61b86b68c7082ae21b4cc9bc88cc36d1c13stefan@webrtc.org render_wait_event_->Reset(); 54e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org if (master_) { 55e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org state_ = kReceiving; 56e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } else { 57e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org state_ = kPassive; 58e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 59b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 60b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 61e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgint32_t VCMReceiver::Initialize() { 62e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org Reset(); 6351d53aa7281e0b96b2d95deff3e03ad4a978e327stefan@webrtc.org CriticalSectionScoped cs(crit_sect_); 64e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org if (!master_) { 65bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org SetNackMode(kNoNack, -1, -1); 66e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 67e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org return VCM_OK; 68b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 69b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 70e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgvoid VCMReceiver::UpdateRtt(uint32_t rtt) { 71e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org jitter_buffer_.UpdateRtt(rtt); 72b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 73b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 74bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.orgint32_t VCMReceiver::InsertPacket(const VCMPacket& packet, 75bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org uint16_t frame_width, 76e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org uint16_t frame_height) { 773740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org // Insert the packet into the jitter buffer. The packet can either be empty or 783740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org // contain media at this point. 793740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org bool retransmitted = false; 803740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org const VCMFrameBufferEnum ret = jitter_buffer_.InsertPacket(packet, 813740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org &retransmitted); 823740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org if (ret == kOldPacket) { 83b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return VCM_OK; 843740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org } else if (ret == kFlushIndicator) { 853740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org return VCM_FLUSH_INDICATOR; 863740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org } else if (ret < 0) { 873740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org return VCM_JITTER_BUFFER_ERROR; 88e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 893740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org if (ret == kCompleteSession && !retransmitted) { 903740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org // We don't want to include timestamps which have suffered from 913740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org // retransmission here, since we compensate with extra retransmission 923740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org // delay within the jitter estimate. 933740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds()); 943740808580c6020ac61ac11a8729fa0a6a08223cstefan@webrtc.org } 95e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org return VCM_OK; 96b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 97b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 98e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgVCMEncodedFrame* VCMReceiver::FrameForDecoding( 99e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org uint16_t max_wait_time_ms, 100e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org int64_t& next_render_time_ms, 101e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org bool render_timing, 102e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org VCMReceiver* dual_receiver) { 1031bb2146351979b6610107419b2a9c86cca2692a3stefan@webrtc.org const int64_t start_time_ms = clock_->TimeInMilliseconds(); 104933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org uint32_t frame_timestamp = 0; 105933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // Exhaust wait time to get a complete frame for decoding. 106933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org bool found_frame = jitter_buffer_.NextCompleteTimestamp( 107933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org max_wait_time_ms, &frame_timestamp); 108e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org 109933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org if (!found_frame) { 110933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // Get an incomplete frame when enabled. 111e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org const bool dual_receiver_enabled_and_passive = (dual_receiver != NULL && 112e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org dual_receiver->State() == kPassive && 113bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org dual_receiver->NackMode() == kNack); 114e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org if (dual_receiver_enabled_and_passive && 115e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org !jitter_buffer_.CompleteSequenceWithNextFrame()) { 116e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org // Jitter buffer state might get corrupt with this frame. 117e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org dual_receiver->CopyJitterBufferStateFromReceiver(*this); 118b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 119933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org found_frame = jitter_buffer_.NextMaybeIncompleteTimestamp( 120933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org &frame_timestamp); 121e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 122e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org 123933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org if (!found_frame) { 124933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org return NULL; 125933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org } 12620eb55811a74b965c3e35fb3746b0081af84ecaestefan@webrtc.org 127933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // We have a frame - Set timing and render timestamp. 128e4775746d3e4d88591ed1acce4c82bd6b0754e8amikhal@webrtc.org timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs()); 129933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org const int64_t now_ms = clock_->TimeInMilliseconds(); 130933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org timing_->UpdateCurrentDelay(frame_timestamp); 131933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org next_render_time_ms = timing_->RenderTimeMs(frame_timestamp, now_ms); 132933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // Check render timing. 133933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org bool timing_error = false; 134933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // Assume that render timing errors are due to changes in the video stream. 135933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org if (next_render_time_ms < 0) { 136933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org timing_error = true; 1372a25b6c685c7d6323a8fdaf514a7b36f4f2d827dpbos@webrtc.org } else if (std::abs(next_render_time_ms - now_ms) > max_video_delay_ms_) { 1388edcccef11eabb0dd98a061c9d640b2b7743609estefan@webrtc.org int frame_delay = static_cast<int>(std::abs(next_render_time_ms - now_ms)); 1398edcccef11eabb0dd98a061c9d640b2b7743609estefan@webrtc.org LOG(LS_WARNING) << "A frame about to be decoded is out of the configured " 1408edcccef11eabb0dd98a061c9d640b2b7743609estefan@webrtc.org << "delay bounds (" << frame_delay << " > " 1418edcccef11eabb0dd98a061c9d640b2b7743609estefan@webrtc.org << max_video_delay_ms_ 1428edcccef11eabb0dd98a061c9d640b2b7743609estefan@webrtc.org << "). Resetting the video jitter buffer."; 143933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org timing_error = true; 144933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org } else if (static_cast<int>(timing_->TargetVideoDelay()) > 145933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org max_video_delay_ms_) { 1468edcccef11eabb0dd98a061c9d640b2b7743609estefan@webrtc.org LOG(LS_WARNING) << "The video target delay has grown larger than " 1478edcccef11eabb0dd98a061c9d640b2b7743609estefan@webrtc.org << max_video_delay_ms_ << " ms. Resetting jitter buffer."; 148933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org timing_error = true; 149e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 150b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 151933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org if (timing_error) { 152933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // Timing error => reset timing and flush the jitter buffer. 153933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org jitter_buffer_.Flush(); 154c7979e04f754f35193be65b2baf7b8b45bfcb57astefan@webrtc.org timing_->Reset(); 155e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org return NULL; 156e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 157933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org 158933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org if (!render_timing) { 159933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // Decode frame as close as possible to the render timestamp. 160933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org const int32_t available_wait_time = max_wait_time_ms - 161933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org static_cast<int32_t>(clock_->TimeInMilliseconds() - start_time_ms); 162933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org uint16_t new_max_wait_time = static_cast<uint16_t>( 163933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org VCM_MAX(available_wait_time, 0)); 164933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org uint32_t wait_time_ms = timing_->MaxWaitingTime( 165933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org next_render_time_ms, clock_->TimeInMilliseconds()); 166933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org if (new_max_wait_time < wait_time_ms) { 167933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // We're not allowed to wait until the frame is supposed to be rendered, 168933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // waiting as long as we're allowed to avoid busy looping, and then return 169933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // NULL. Next call to this function might return the frame. 170933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org render_wait_event_->Wait(max_wait_time_ms); 171933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org return NULL; 172b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 173933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // Wait until it's time to render. 174933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org render_wait_event_->Wait(wait_time_ms); 175933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org } 176b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 177933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // Extract the frame from the jitter buffer and set the render time. 178933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org VCMEncodedFrame* frame = jitter_buffer_.ExtractAndSetDecode(frame_timestamp); 17938fb7b0c8a7fd884db2f648d7a3c2c91ea4fbd4amikhal@webrtc.org if (frame == NULL) { 18038fb7b0c8a7fd884db2f648d7a3c2c91ea4fbd4amikhal@webrtc.org return NULL; 18138fb7b0c8a7fd884db2f648d7a3c2c91ea4fbd4amikhal@webrtc.org } 182933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org frame->SetRenderTime(next_render_time_ms); 1839c0f14d2cd43e388b5136f0ced7951ee58610e86hclam@chromium.org TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame->TimeStamp(), 1849c0f14d2cd43e388b5136f0ced7951ee58610e86hclam@chromium.org "SetRenderTS", "render_time", next_render_time_ms); 185933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org if (dual_receiver != NULL) { 186933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org dual_receiver->UpdateState(*frame); 187933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org } 188933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org if (!frame->Complete()) { 189933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // Update stats for incomplete frames. 190933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org bool retransmitted = false; 191933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org const int64_t last_packet_time_ms = 192933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org jitter_buffer_.LastPacketTime(frame, &retransmitted); 193933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org if (last_packet_time_ms >= 0 && !retransmitted) { 194933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // We don't want to include timestamps which have suffered from 195933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // retransmission here, since we compensate with extra retransmission 196933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // delay within the jitter estimate. 197933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org timing_->IncomingTimestamp(frame_timestamp, last_packet_time_ms); 198933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org } 199e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 200e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org return frame; 201b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 202b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 203e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgvoid VCMReceiver::ReleaseFrame(VCMEncodedFrame* frame) { 204e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org jitter_buffer_.ReleaseFrame(frame); 205b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 206b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 207e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgvoid VCMReceiver::ReceiveStatistics(uint32_t* bitrate, 208e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org uint32_t* framerate) { 209e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org assert(bitrate); 210e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org assert(framerate); 211e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org jitter_buffer_.IncomingRateStatistics(framerate, bitrate); 212b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 213b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 214e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgvoid VCMReceiver::ReceivedFrameCount(VCMFrameCount* frame_count) const { 215e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org assert(frame_count); 2165fdd10a56c3d7dcf2ea3cb2cd1118f616b783d24sprang@webrtc.org std::map<FrameType, uint32_t> counts(jitter_buffer_.FrameStatistics()); 2175fdd10a56c3d7dcf2ea3cb2cd1118f616b783d24sprang@webrtc.org frame_count->numDeltaFrames = counts[kVideoFrameDelta]; 2185fdd10a56c3d7dcf2ea3cb2cd1118f616b783d24sprang@webrtc.org frame_count->numKeyFrames = counts[kVideoFrameKey]; 219b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 220b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 221e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orguint32_t VCMReceiver::DiscardedPackets() const { 222e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org return jitter_buffer_.num_discarded_packets(); 223b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 224b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 225bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.orgvoid VCMReceiver::SetNackMode(VCMNackMode nackMode, 226bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org int low_rtt_nack_threshold_ms, 227bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org int high_rtt_nack_threshold_ms) { 228e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org CriticalSectionScoped cs(crit_sect_); 229e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org // Default to always having NACK enabled in hybrid mode. 230bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org jitter_buffer_.SetNackMode(nackMode, low_rtt_nack_threshold_ms, 231bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org high_rtt_nack_threshold_ms); 232e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org if (!master_) { 233e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org state_ = kPassive; // The dual decoder defaults to passive. 234e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 235b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 236b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 2377fff32c808707eb6810d1057747069d7afce9939stefan@webrtc.orgvoid VCMReceiver::SetNackSettings(size_t max_nack_list_size, 23806ad384100b8b493e4a3f37877caac520189fec3stefan@webrtc.org int max_packet_age_to_nack, 23906ad384100b8b493e4a3f37877caac520189fec3stefan@webrtc.org int max_incomplete_time_ms) { 2407fff32c808707eb6810d1057747069d7afce9939stefan@webrtc.org jitter_buffer_.SetNackSettings(max_nack_list_size, 24106ad384100b8b493e4a3f37877caac520189fec3stefan@webrtc.org max_packet_age_to_nack, 24206ad384100b8b493e4a3f37877caac520189fec3stefan@webrtc.org max_incomplete_time_ms); 2437fff32c808707eb6810d1057747069d7afce9939stefan@webrtc.org} 2447fff32c808707eb6810d1057747069d7afce9939stefan@webrtc.org 245e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgVCMNackMode VCMReceiver::NackMode() const { 246e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org CriticalSectionScoped cs(crit_sect_); 247e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org return jitter_buffer_.nack_mode(); 248b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 249b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 250e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgVCMNackStatus VCMReceiver::NackList(uint16_t* nack_list, 251bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org uint16_t size, 252bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org uint16_t* nack_list_length) { 253bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org bool request_key_frame = false; 254bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org uint16_t* internal_nack_list = jitter_buffer_.GetNackList( 255bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org nack_list_length, &request_key_frame); 2568edcccef11eabb0dd98a061c9d640b2b7743609estefan@webrtc.org assert(*nack_list_length <= size); 257bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org if (internal_nack_list != NULL && *nack_list_length > 0) { 258bb78b2c7ce8cc4445811d0e881890a29b14a4f2bstefan@webrtc.org memcpy(nack_list, internal_nack_list, *nack_list_length * sizeof(uint16_t)); 259e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 26006ad384100b8b493e4a3f37877caac520189fec3stefan@webrtc.org if (request_key_frame) { 26106ad384100b8b493e4a3f37877caac520189fec3stefan@webrtc.org return kNackKeyFrameRequest; 26206ad384100b8b493e4a3f37877caac520189fec3stefan@webrtc.org } 263e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org return kNackOk; 264b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 265b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 266e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org// Decide whether we should change decoder state. This should be done if the 267e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org// dual decoder has caught up with the decoder decoding with packet losses. 268e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgbool VCMReceiver::DualDecoderCaughtUp(VCMEncodedFrame* dual_frame, 269e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org VCMReceiver& dual_receiver) const { 270e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org if (dual_frame == NULL) { 271e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org return false; 272e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 273e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org if (jitter_buffer_.LastDecodedTimestamp() == dual_frame->TimeStamp()) { 274e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org dual_receiver.UpdateState(kWaitForPrimaryDecode); 275e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org return true; 276e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 277e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org return false; 278b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 279b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 280e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgvoid VCMReceiver::CopyJitterBufferStateFromReceiver( 281e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org const VCMReceiver& receiver) { 282e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org jitter_buffer_.CopyFrom(receiver.jitter_buffer_); 283b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 284b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 285e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgVCMReceiverState VCMReceiver::State() const { 286e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org CriticalSectionScoped cs(crit_sect_); 287e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org return state_; 288b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 289b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 2909e8a66c236b96d53c6a70f93f7dc78f88ea52540mikhal@webrtc.orgvoid VCMReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) { 2919e8a66c236b96d53c6a70f93f7dc78f88ea52540mikhal@webrtc.org jitter_buffer_.SetDecodeErrorMode(decode_error_mode); 2921886a048dbe72297b469be699a9750ae4a986967mikhal@webrtc.org} 2931886a048dbe72297b469be699a9750ae4a986967mikhal@webrtc.org 2947b2147f8975308f753380cf0248b84a733970a10agalusza@google.comVCMDecodeErrorMode VCMReceiver::DecodeErrorMode() const { 2957b2147f8975308f753380cf0248b84a733970a10agalusza@google.com return jitter_buffer_.decode_error_mode(); 2961886a048dbe72297b469be699a9750ae4a986967mikhal@webrtc.org} 2971886a048dbe72297b469be699a9750ae4a986967mikhal@webrtc.org 2989d6fcb37be4c9f49e1bb0b32bb01314c31f5e1ddmikhal@webrtc.orgint VCMReceiver::SetMinReceiverDelay(int desired_delay_ms) { 2999d6fcb37be4c9f49e1bb0b32bb01314c31f5e1ddmikhal@webrtc.org CriticalSectionScoped cs(crit_sect_); 3009d6fcb37be4c9f49e1bb0b32bb01314c31f5e1ddmikhal@webrtc.org if (desired_delay_ms < 0 || desired_delay_ms > kMaxReceiverDelayMs) { 3019d6fcb37be4c9f49e1bb0b32bb01314c31f5e1ddmikhal@webrtc.org return -1; 3029d6fcb37be4c9f49e1bb0b32bb01314c31f5e1ddmikhal@webrtc.org } 3039d6fcb37be4c9f49e1bb0b32bb01314c31f5e1ddmikhal@webrtc.org max_video_delay_ms_ = desired_delay_ms + kMaxVideoDelayMs; 304e7afdc72e3f200b2f29cb7c7847fbc157dca72ecmikhal@webrtc.org // Initializing timing to the desired delay. 305e4775746d3e4d88591ed1acce4c82bd6b0754e8amikhal@webrtc.org timing_->set_min_playout_delay(desired_delay_ms); 3069d6fcb37be4c9f49e1bb0b32bb01314c31f5e1ddmikhal@webrtc.org return 0; 3079d6fcb37be4c9f49e1bb0b32bb01314c31f5e1ddmikhal@webrtc.org} 3089d6fcb37be4c9f49e1bb0b32bb01314c31f5e1ddmikhal@webrtc.org 30940bd7448cb81dea34512c6970864a710a75da666mikhal@webrtc.orgint VCMReceiver::RenderBufferSizeMs() { 310933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org uint32_t timestamp_start = 0u; 311933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org uint32_t timestamp_end = 0u; 312933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // Render timestamps are computed just prior to decoding. Therefore this is 313933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // only an estimate based on frames' timestamps and current timing state. 314933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org jitter_buffer_.RenderBufferSize(×tamp_start, ×tamp_end); 315933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org if (timestamp_start == timestamp_end) { 316933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org return 0; 317933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org } 318933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // Update timing. 319933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org const int64_t now_ms = clock_->TimeInMilliseconds(); 320e4775746d3e4d88591ed1acce4c82bd6b0754e8amikhal@webrtc.org timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs()); 321933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org // Get render timestamps. 322933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org uint32_t render_start = timing_->RenderTimeMs(timestamp_start, now_ms); 323933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org uint32_t render_end = timing_->RenderTimeMs(timestamp_end, now_ms); 324933f88591c8593007a8186f75d5e941310fb4051mikhal@webrtc.org return render_end - render_start; 32540bd7448cb81dea34512c6970864a710a75da666mikhal@webrtc.org} 32640bd7448cb81dea34512c6970864a710a75da666mikhal@webrtc.org 327e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgvoid VCMReceiver::UpdateState(VCMReceiverState new_state) { 328e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org CriticalSectionScoped cs(crit_sect_); 329e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org assert(!(state_ == kPassive && new_state == kWaitForPrimaryDecode)); 330e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org state_ = new_state; 331b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 332b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 333e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.orgvoid VCMReceiver::UpdateState(const VCMEncodedFrame& frame) { 334e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org if (jitter_buffer_.nack_mode() == kNoNack) { 335e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org // Dual decoder mode has not been enabled. 336e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org return; 337e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 338e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org // Update the dual receiver state. 339e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org if (frame.Complete() && frame.FrameType() == kVideoFrameKey) { 340e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org UpdateState(kPassive); 341e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 342e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org if (State() == kWaitForPrimaryDecode && 343e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org frame.Complete() && !frame.MissingFrame()) { 344e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org UpdateState(kPassive); 345e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 346e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org if (frame.MissingFrame() || !frame.Complete()) { 347e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org // State was corrupted, enable dual receiver. 348e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org UpdateState(kReceiving); 349e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org } 350b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 351e9dc60a50e0ae5da68fb4c3b1fad402f69bba22fstefan@webrtc.org} // namespace webrtc 352