129d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org/* 229d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 329d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org * 429d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org * Use of this source code is governed by a BSD-style license 529d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org * that can be found in the LICENSE file in the root of the source 629d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org * tree. An additional intellectual property rights grant can be found 729d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org * in the file PATENTS. All contributing project authors may 829d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org * be found in the AUTHORS file in the root of the source tree. 929d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org */ 1029d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 1116e03b7bd8b88ba569987e20a7f29061f91a3d0dpbos@webrtc.org#include "webrtc/video/video_receive_stream.h" 1229d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 1312dc1a38ca54a000e4fecfbc6d41138b895c9ca5pbos@webrtc.org#include <stdlib.h> 1429d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 15d1590b2571c4cb33416e14c92e4f2dfed42ec3d4mflodman#include <set> 16b429e516a98d2dee0c57d3263f6d21633939b564mflodman@webrtc.org#include <string> 17b429e516a98d2dee0c57d3263f6d21633939b564mflodman@webrtc.org 180d852d5c27a759fe7aadc500bd7b3cadfae3deb8pbos@webrtc.org#include "webrtc/base/checks.h" 19415d2cd7454d93b3727fce9147090a24e4c3ccbaPeter Boström#include "webrtc/base/logging.h" 200c478b3d75be3c026e68f03a11cb558c3655c926mflodman#include "webrtc/call/congestion_controller.h" 2129d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" 2298f53510b222f71fdd8b799b2f33737ceeb28c61Henrik Kjellander#include "webrtc/system_wrappers/include/clock.h" 237623ce4aeb9130c937ba5836490cbb3a35679e79Peter Boström#include "webrtc/video/call_stats.h" 2409315705b9caf3bff455e3515b9bd99492a7c3e3sprang@webrtc.org#include "webrtc/video/receive_statistics_proxy.h" 2516e03b7bd8b88ba569987e20a7f29061f91a3d0dpbos@webrtc.org#include "webrtc/video_receive_stream.h" 2629d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 2729d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.orgnamespace webrtc { 28a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman 29a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodmanstatic bool UseSendSideBwe(const std::vector<RtpExtension>& extensions) { 30a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman for (const auto& extension : extensions) { 31a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman if (extension.name == RtpExtension::kTransportSequenceNumber) 32a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman return true; 33a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman } 34a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman return false; 35a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman} 36a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman 3732e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.orgstd::string VideoReceiveStream::Decoder::ToString() const { 3832e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org std::stringstream ss; 392b4ce3a501b8d679f84c1ad10317dea5c78fa595pbos@webrtc.org ss << "{decoder: " << (decoder != nullptr ? "(VideoDecoder)" : "nullptr"); 4032e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", payload_type: " << payload_type; 4132e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", payload_name: " << payload_name; 4232e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << '}'; 4332e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org 4432e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org return ss.str(); 4532e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org} 4632e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org 4732e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.orgstd::string VideoReceiveStream::Config::ToString() const { 4832e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org std::stringstream ss; 4932e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << "{decoders: ["; 5032e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org for (size_t i = 0; i < decoders.size(); ++i) { 5132e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << decoders[i].ToString(); 5232e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org if (i != decoders.size() - 1) 5332e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", "; 5432e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org } 5532e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ']'; 5632e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", rtp: " << rtp.ToString(); 572b4ce3a501b8d679f84c1ad10317dea5c78fa595pbos@webrtc.org ss << ", renderer: " << (renderer != nullptr ? "(renderer)" : "nullptr"); 5832e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", render_delay_ms: " << render_delay_ms; 598fc7fa798f7a36955f1b933980401afad2aff592pbos if (!sync_group.empty()) 608fc7fa798f7a36955f1b933980401afad2aff592pbos ss << ", sync_group: " << sync_group; 6132e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", pre_decode_callback: " 622b4ce3a501b8d679f84c1ad10317dea5c78fa595pbos@webrtc.org << (pre_decode_callback != nullptr ? "(EncodedFrameObserver)" : "nullptr"); 6332e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", pre_render_callback: " 642b4ce3a501b8d679f84c1ad10317dea5c78fa595pbos@webrtc.org << (pre_render_callback != nullptr ? "(I420FrameCallback)" : "nullptr"); 6532e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", target_delay_ms: " << target_delay_ms; 6632e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << '}'; 6732e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org 6832e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org return ss.str(); 6932e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org} 7032e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org 7132e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.orgstd::string VideoReceiveStream::Config::Rtp::ToString() const { 7232e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org std::stringstream ss; 7332e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << "{remote_ssrc: " << remote_ssrc; 7432e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", local_ssrc: " << local_ssrc; 75da903eaabbb6c6830efcafc3c2ade1d36f511e43pbos ss << ", rtcp_mode: " 76da903eaabbb6c6830efcafc3c2ade1d36f511e43pbos << (rtcp_mode == RtcpMode::kCompound ? "RtcpMode::kCompound" 77da903eaabbb6c6830efcafc3c2ade1d36f511e43pbos : "RtcpMode::kReducedSize"); 7832e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", rtcp_xr: "; 7932e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << "{receiver_reference_time_report: " 8032e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org << (rtcp_xr.receiver_reference_time_report ? "on" : "off"); 8132e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << '}'; 8232e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", remb: " << (remb ? "on" : "off"); 8343edf0ffb91a50e2efa01c7befe4d188a7e30ea2stefan ss << ", transport_cc: " << (transport_cc ? "on" : "off"); 8432e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", nack: {rtp_history_ms: " << nack.rtp_history_ms << '}'; 8532e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", fec: " << fec.ToString(); 8632e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", rtx: {"; 8732e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org for (auto& kv : rtx) { 8832e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << kv.first << " -> "; 8932e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << "{ssrc: " << kv.second.ssrc; 9032e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", payload_type: " << kv.second.payload_type; 9132e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << '}'; 9232e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org } 9332e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << '}'; 9432e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", extensions: ["; 9532e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org for (size_t i = 0; i < extensions.size(); ++i) { 9632e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << extensions[i].ToString(); 9732e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org if (i != extensions.size() - 1) 9832e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ", "; 9932e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org } 10032e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << ']'; 10132e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org ss << '}'; 10232e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org return ss.str(); 10332e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org} 10432e852858101c3565cfc79cdda9310a3336d95a0pbos@webrtc.org 10529d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.orgnamespace internal { 106776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.orgnamespace { 1079e4e524f3894c7eeee8a53bd3cbf21d27b5efc8cpbos@webrtc.org 108776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.orgVideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) { 109776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org VideoCodec codec; 110776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org memset(&codec, 0, sizeof(codec)); 111776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org 112776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org codec.plType = decoder.payload_type; 113d1590b2571c4cb33416e14c92e4f2dfed42ec3d4mflodman strncpy(codec.plName, decoder.payload_name.c_str(), sizeof(codec.plName)); 114776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org if (decoder.payload_name == "VP8") { 115776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org codec.codecType = kVideoCodecVP8; 11605cfcd34693d86de7a1a481f071eae561361588aivica } else if (decoder.payload_name == "VP9") { 11705cfcd34693d86de7a1a481f071eae561361588aivica codec.codecType = kVideoCodecVP9; 118776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org } else if (decoder.payload_name == "H264") { 119776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org codec.codecType = kVideoCodecH264; 120776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org } else { 121776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org codec.codecType = kVideoCodecGeneric; 122776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org } 123776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org 124776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org if (codec.codecType == kVideoCodecVP8) { 125776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings(); 12605cfcd34693d86de7a1a481f071eae561361588aivica } else if (codec.codecType == kVideoCodecVP9) { 12705cfcd34693d86de7a1a481f071eae561361588aivica codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings(); 128776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org } else if (codec.codecType == kVideoCodecH264) { 129776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings(); 130776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org } 131776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org 132776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org codec.width = 320; 133776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org codec.height = 180; 134776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org codec.startBitrate = codec.minBitrate = codec.maxBitrate = 135776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org Call::Config::kDefaultStartBitrateBps / 1000; 136776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org 137776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org return codec; 138776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org} 139776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org} // namespace 14029d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 1410c478b3d75be3c026e68f03a11cb558c3655c926mflodmanVideoReceiveStream::VideoReceiveStream( 1420c478b3d75be3c026e68f03a11cb558c3655c926mflodman int num_cpu_cores, 1430c478b3d75be3c026e68f03a11cb558c3655c926mflodman CongestionController* congestion_controller, 1440c478b3d75be3c026e68f03a11cb558c3655c926mflodman const VideoReceiveStream::Config& config, 1450c478b3d75be3c026e68f03a11cb558c3655c926mflodman webrtc::VoiceEngine* voice_engine, 1460c478b3d75be3c026e68f03a11cb558c3655c926mflodman ProcessThread* process_thread, 1470c478b3d75be3c026e68f03a11cb558c3655c926mflodman CallStats* call_stats) 1484fbae2b79134572135d9d5fe35a7d1ccdeea3a4dsolenberg : transport_adapter_(config.rtcp_send_transport), 1494070935f4fb5b9fb2df246d7073fe0ba7e350791sprang@webrtc.org encoded_frame_proxy_(config.pre_decode_callback), 1504070935f4fb5b9fb2df246d7073fe0ba7e350791sprang@webrtc.org config_(config), 15109315705b9caf3bff455e3515b9bd99492a7c3e3sprang@webrtc.org clock_(Clock::GetRealTimeClock()), 1520c478b3d75be3c026e68f03a11cb558c3655c926mflodman congestion_controller_(congestion_controller), 153e37870297fc45f1253dff4b1c59c85a50d2a8a97mflodman call_stats_(call_stats) { 154a2f30deea342896ee40cc4d90567f091efbe0fc9pbos LOG(LS_INFO) << "VideoReceiveStream: " << config_.ToString(); 15529d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 15643edf0ffb91a50e2efa01c7befe4d188a7e30ea2stefan bool send_side_bwe = 15743edf0ffb91a50e2efa01c7befe4d188a7e30ea2stefan config.rtp.transport_cc && UseSendSideBwe(config_.rtp.extensions); 158a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman 159a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman RemoteBitrateEstimator* bitrate_estimator = 1600c478b3d75be3c026e68f03a11cb558c3655c926mflodman congestion_controller_->GetRemoteBitrateEstimator(send_side_bwe); 161a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman 162a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman vie_channel_.reset(new ViEChannel( 163e37870297fc45f1253dff4b1c59c85a50d2a8a97mflodman num_cpu_cores, &transport_adapter_, process_thread, nullptr, 1640a41893e36148642d8f37fc91738e42eca47fef8mflodman nullptr, nullptr, bitrate_estimator, call_stats_->rtcp_rtt_stats(), 1650c478b3d75be3c026e68f03a11cb558c3655c926mflodman congestion_controller_->pacer(), congestion_controller_->packet_router(), 1660c478b3d75be3c026e68f03a11cb558c3655c926mflodman 1, false)); 167a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman 168a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman RTC_CHECK(vie_channel_->Init() == 0); 169a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman 170a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman // Register the channel to receive stats updates. 171e37870297fc45f1253dff4b1c59c85a50d2a8a97mflodman call_stats_->RegisterStatsObserver(vie_channel_->GetStatsObserver()); 172a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman 173bbb07e69e55761ca43b38f929f3e39f4bba70ef1pbos@webrtc.org // TODO(pbos): This is not fine grained enough... 174ba8c15b857c0f341d9c1e02d41b6ccd56f9f1030pbos vie_channel_->SetProtectionMode(config_.rtp.nack.rtp_history_ms > 0, false, 175ba8c15b857c0f341d9c1e02d41b6ccd56f9f1030pbos -1, -1); 176da903eaabbb6c6830efcafc3c2ade1d36f511e43pbos RTC_DCHECK(config_.rtp.rtcp_mode != RtcpMode::kOff) 177da903eaabbb6c6830efcafc3c2ade1d36f511e43pbos << "A stream should not be configured with RTCP disabled. This value is " 178da903eaabbb6c6830efcafc3c2ade1d36f511e43pbos "reserved for internal usage."; 179da903eaabbb6c6830efcafc3c2ade1d36f511e43pbos vie_channel_->SetRTCPMode(config_.rtp.rtcp_mode); 180bbb07e69e55761ca43b38f929f3e39f4bba70ef1pbos@webrtc.org 18191d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_DCHECK(config_.rtp.remote_ssrc != 0); 182c279a5d72c885b1a1737018ee26dc7c0475a38bfpbos@webrtc.org // TODO(pbos): What's an appropriate local_ssrc for receive-only streams? 18391d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_DCHECK(config_.rtp.local_ssrc != 0); 18491d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_DCHECK(config_.rtp.remote_ssrc != config_.rtp.local_ssrc); 185b613b5ab2b03041942f04fd892e2ad5a4f9de027pbos@webrtc.org 18676c53d36bc455fe89ca1f860d5171633198fe907Peter Boström vie_channel_->SetSSRC(config_.rtp.local_ssrc, kViEStreamTypeNormal, 0); 187c279a5d72c885b1a1737018ee26dc7c0475a38bfpbos@webrtc.org // TODO(pbos): Support multiple RTX, per video payload. 188c279a5d72c885b1a1737018ee26dc7c0475a38bfpbos@webrtc.org Config::Rtp::RtxMap::const_iterator it = config_.rtp.rtx.begin(); 189e62202fedf57b74cc263246c0586ee353978caf8Shao Changbin for (; it != config_.rtp.rtx.end(); ++it) { 19091d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_DCHECK(it->second.ssrc != 0); 19191d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_DCHECK(it->second.payload_type != 0); 192c279a5d72c885b1a1737018ee26dc7c0475a38bfpbos@webrtc.org 19376c53d36bc455fe89ca1f860d5171633198fe907Peter Boström vie_channel_->SetRemoteSSRCType(kViEStreamTypeRtx, it->second.ssrc); 194e62202fedf57b74cc263246c0586ee353978caf8Shao Changbin vie_channel_->SetRtxReceivePayloadType(it->second.payload_type, it->first); 195c279a5d72c885b1a1737018ee26dc7c0475a38bfpbos@webrtc.org } 19665220a70a38ffe252b587775c5e9104606ab7c2cnoahric // TODO(holmer): When Chrome no longer depends on this being false by default, 19765220a70a38ffe252b587775c5e9104606ab7c2cnoahric // always use the mapping and remove this whole codepath. 19865220a70a38ffe252b587775c5e9104606ab7c2cnoahric vie_channel_->SetUseRtxPayloadMappingOnRestore( 19965220a70a38ffe252b587775c5e9104606ab7c2cnoahric config_.rtp.use_rtx_payload_mapping_on_restore); 200c279a5d72c885b1a1737018ee26dc7c0475a38bfpbos@webrtc.org 2010c478b3d75be3c026e68f03a11cb558c3655c926mflodman congestion_controller_->SetChannelRembStatus(false, config_.rtp.remb, 2020c478b3d75be3c026e68f03a11cb558c3655c926mflodman vie_channel_->rtp_rtcp()); 20329d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 2045ab756703ea32f2c2ff9878d6eae628c7380bc14pbos@webrtc.org for (size_t i = 0; i < config_.rtp.extensions.size(); ++i) { 2055ab756703ea32f2c2ff9878d6eae628c7380bc14pbos@webrtc.org const std::string& extension = config_.rtp.extensions[i].name; 2065ab756703ea32f2c2ff9878d6eae628c7380bc14pbos@webrtc.org int id = config_.rtp.extensions[i].id; 20723914fe756903353eae13fffc868d2c84f51f06fPeter Boström // One-byte-extension local identifiers are in the range 1-14 inclusive. 20891d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_DCHECK_GE(id, 1); 20991d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_DCHECK_LE(id, 14); 2105ab756703ea32f2c2ff9878d6eae628c7380bc14pbos@webrtc.org if (extension == RtpExtension::kTOffset) { 21191d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_CHECK_EQ(0, vie_channel_->SetReceiveTimestampOffsetStatus(true, id)); 2125ab756703ea32f2c2ff9878d6eae628c7380bc14pbos@webrtc.org } else if (extension == RtpExtension::kAbsSendTime) { 21391d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_CHECK_EQ(0, vie_channel_->SetReceiveAbsoluteSendTimeStatus(true, id)); 214fdd10579496123c9a7fdc0bf185e2a26a12ed340guoweis@webrtc.org } else if (extension == RtpExtension::kVideoRotation) { 21591d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_CHECK_EQ(0, vie_channel_->SetReceiveVideoRotationStatus(true, id)); 216867fb5224e1ba6a1c2cd523c005499a93ed61a08sprang } else if (extension == RtpExtension::kTransportSequenceNumber) { 21791d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_CHECK_EQ(0, 21891d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg vie_channel_->SetReceiveTransportSequenceNumber(true, id)); 2195ab756703ea32f2c2ff9878d6eae628c7380bc14pbos@webrtc.org } else { 2202b4ce3a501b8d679f84c1ad10317dea5c78fa595pbos@webrtc.org RTC_NOTREACHED() << "Unsupported RTP extension."; 2215ab756703ea32f2c2ff9878d6eae628c7380bc14pbos@webrtc.org } 2225ab756703ea32f2c2ff9878d6eae628c7380bc14pbos@webrtc.org } 2235ab756703ea32f2c2ff9878d6eae628c7380bc14pbos@webrtc.org 2242a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org if (config_.rtp.fec.ulpfec_payload_type != -1) { 2252a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org // ULPFEC without RED doesn't make sense. 22691d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_DCHECK(config_.rtp.fec.red_payload_type != -1); 2272a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org VideoCodec codec; 2282a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org memset(&codec, 0, sizeof(codec)); 2292a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org codec.codecType = kVideoCodecULPFEC; 230d1590b2571c4cb33416e14c92e4f2dfed42ec3d4mflodman strncpy(codec.plName, "ulpfec", sizeof(codec.plName)); 2312a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org codec.plType = config_.rtp.fec.ulpfec_payload_type; 23291d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_CHECK_EQ(0, vie_channel_->SetReceiveCodec(codec)); 2332a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org } 2342a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org if (config_.rtp.fec.red_payload_type != -1) { 2352a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org VideoCodec codec; 2362a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org memset(&codec, 0, sizeof(codec)); 2372a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org codec.codecType = kVideoCodecRED; 238d1590b2571c4cb33416e14c92e4f2dfed42ec3d4mflodman strncpy(codec.plName, "red", sizeof(codec.plName)); 2392a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org codec.plType = config_.rtp.fec.red_payload_type; 24091d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_CHECK_EQ(0, vie_channel_->SetReceiveCodec(codec)); 241e62202fedf57b74cc263246c0586ee353978caf8Shao Changbin if (config_.rtp.fec.red_rtx_payload_type != -1) { 242e62202fedf57b74cc263246c0586ee353978caf8Shao Changbin vie_channel_->SetRtxReceivePayloadType( 243e62202fedf57b74cc263246c0586ee353978caf8Shao Changbin config_.rtp.fec.red_rtx_payload_type, 244e62202fedf57b74cc263246c0586ee353978caf8Shao Changbin config_.rtp.fec.red_payload_type); 245e62202fedf57b74cc263246c0586ee353978caf8Shao Changbin } 2462a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org } 2472a034988257074183615615f0cc7f59f3411a21cpbos@webrtc.org 24876c53d36bc455fe89ca1f860d5171633198fe907Peter Boström if (config.rtp.rtcp_xr.receiver_reference_time_report) 24976c53d36bc455fe89ca1f860d5171633198fe907Peter Boström vie_channel_->SetRtcpXrRrtrStatus(true); 25076c53d36bc455fe89ca1f860d5171633198fe907Peter Boström 25155707692105a4765f8f321ec7c30a1034d03d23apbos@webrtc.org stats_proxy_.reset( 2521d0fa5d352fe12092201fade249905c7e1ff974bpbos@webrtc.org new ReceiveStatisticsProxy(config_.rtp.remote_ssrc, clock_)); 25309315705b9caf3bff455e3515b9bd99492a7c3e3sprang@webrtc.org 254f42376c60111edba6f29060bf3dd79e75d8dbb97pbos vie_channel_->RegisterReceiveStatisticsProxy(stats_proxy_.get()); 25576c53d36bc455fe89ca1f860d5171633198fe907Peter Boström vie_channel_->RegisterReceiveChannelRtcpStatisticsCallback( 25676c53d36bc455fe89ca1f860d5171633198fe907Peter Boström stats_proxy_.get()); 25776c53d36bc455fe89ca1f860d5171633198fe907Peter Boström vie_channel_->RegisterReceiveChannelRtpStatisticsCallback(stats_proxy_.get()); 25876c53d36bc455fe89ca1f860d5171633198fe907Peter Boström vie_channel_->RegisterRtcpPacketTypeCounterObserver(stats_proxy_.get()); 25909315705b9caf3bff455e3515b9bd99492a7c3e3sprang@webrtc.org 26091d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_DCHECK(!config_.decoders.empty()); 261521af4e344678ce9dcf996341af6ba8056e1e147Peter Boström std::set<int> decoder_payload_types; 262776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org for (size_t i = 0; i < config_.decoders.size(); ++i) { 263776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org const Decoder& decoder = config_.decoders[i]; 264795dbe4e0fa78333d52fe39edb08a0b13a281c34Peter Boström RTC_CHECK(decoder.decoder); 265521af4e344678ce9dcf996341af6ba8056e1e147Peter Boström RTC_CHECK(decoder_payload_types.find(decoder.payload_type) == 266521af4e344678ce9dcf996341af6ba8056e1e147Peter Boström decoder_payload_types.end()) 267521af4e344678ce9dcf996341af6ba8056e1e147Peter Boström << "Duplicate payload type (" << decoder.payload_type 268521af4e344678ce9dcf996341af6ba8056e1e147Peter Boström << ") for different decoders."; 269521af4e344678ce9dcf996341af6ba8056e1e147Peter Boström decoder_payload_types.insert(decoder.payload_type); 270796cfaf7f76aa740cc7f4bb2c94f88637e475324perkj vie_channel_->RegisterExternalDecoder(decoder.payload_type, 271796cfaf7f76aa740cc7f4bb2c94f88637e475324perkj decoder.decoder); 272776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org 273776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org VideoCodec codec = CreateDecoderVideoCodec(decoder); 274776e6f289c7396a1143b8b36b03f88b08ac8cba3pbos@webrtc.org 27591d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_CHECK_EQ(0, vie_channel_->SetReceiveCodec(codec)); 2760181b5f8dd719f7b1f938d12875a96e59fef44f5pbos@webrtc.org } 2770181b5f8dd719f7b1f938d12875a96e59fef44f5pbos@webrtc.org 278444682acf9804c5fcbddaded9e900ba3cc6921fcqiangchen incoming_video_stream_.reset(new IncomingVideoStream( 279444682acf9804c5fcbddaded9e900ba3cc6921fcqiangchen 0, config.renderer ? config.renderer->SmoothsRenderedFrames() : false)); 280c4188fd3c74688264621393fc622cb81c042c1acPeter Boström incoming_video_stream_->SetExpectedRenderDelay(config.render_delay_ms); 281796cfaf7f76aa740cc7f4bb2c94f88637e475324perkj vie_channel_->SetExpectedRenderDelay(config.render_delay_ms); 282c4188fd3c74688264621393fc622cb81c042c1acPeter Boström incoming_video_stream_->SetExternalCallback(this); 283c4188fd3c74688264621393fc622cb81c042c1acPeter Boström vie_channel_->SetIncomingVideoStream(incoming_video_stream_.get()); 284fe1ef935e7d35b465bd00744cc188f08198265b6pbos@webrtc.org 28586b016027d2d27c62fedd108354a2b1274118ae3asapersson vie_channel_->RegisterPreDecodeImageCallback(this); 28676c53d36bc455fe89ca1f860d5171633198fe907Peter Boström vie_channel_->RegisterPreRenderCallback(this); 28729d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org} 28829d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 28929d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.orgVideoReceiveStream::~VideoReceiveStream() { 290a2f30deea342896ee40cc4d90567f091efbe0fc9pbos LOG(LS_INFO) << "~VideoReceiveStream: " << config_.ToString(); 29152ef9d77386e8d3e40c8d3d07b93f06950aede72Peter Boström incoming_video_stream_->Stop(); 29276c53d36bc455fe89ca1f860d5171633198fe907Peter Boström vie_channel_->RegisterPreRenderCallback(nullptr); 29376c53d36bc455fe89ca1f860d5171633198fe907Peter Boström vie_channel_->RegisterPreDecodeImageCallback(nullptr); 294fe1ef935e7d35b465bd00744cc188f08198265b6pbos@webrtc.org 295e37870297fc45f1253dff4b1c59c85a50d2a8a97mflodman call_stats_->DeregisterStatsObserver(vie_channel_->GetStatsObserver()); 2960c478b3d75be3c026e68f03a11cb558c3655c926mflodman congestion_controller_->SetChannelRembStatus(false, false, 2970c478b3d75be3c026e68f03a11cb558c3655c926mflodman vie_channel_->rtp_rtcp()); 298a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman 299a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman uint32_t remote_ssrc = vie_channel_->GetRemoteSSRC(); 300a20de2030f7f3a3c5e252ccc76a467109f5a93dcmflodman bool send_side_bwe = UseSendSideBwe(config_.rtp.extensions); 3010c478b3d75be3c026e68f03a11cb558c3655c926mflodman congestion_controller_->GetRemoteBitrateEstimator(send_side_bwe)-> 3020c478b3d75be3c026e68f03a11cb558c3655c926mflodman RemoveStream(remote_ssrc); 30329d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org} 30429d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 305a5c8d2c9b39a2d20fead2147e60ed0cd6d62019cpbos@webrtc.orgvoid VideoReceiveStream::Start() { 306d9b9560ee50c236efcb690ee479021b415f7dfd4sprang@webrtc.org transport_adapter_.Enable(); 307c4188fd3c74688264621393fc622cb81c042c1acPeter Boström incoming_video_stream_->Start(); 30876c53d36bc455fe89ca1f860d5171633198fe907Peter Boström vie_channel_->StartReceive(); 30929d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org} 31029d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 311a5c8d2c9b39a2d20fead2147e60ed0cd6d62019cpbos@webrtc.orgvoid VideoReceiveStream::Stop() { 312c4188fd3c74688264621393fc622cb81c042c1acPeter Boström incoming_video_stream_->Stop(); 31376c53d36bc455fe89ca1f860d5171633198fe907Peter Boström vie_channel_->StopReceive(); 314d9b9560ee50c236efcb690ee479021b415f7dfd4sprang@webrtc.org transport_adapter_.Disable(); 31529d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org} 31629d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 3178fc7fa798f7a36955f1b933980401afad2aff592pbosvoid VideoReceiveStream::SetSyncChannel(VoiceEngine* voice_engine, 3188fc7fa798f7a36955f1b933980401afad2aff592pbos int audio_channel_id) { 3198fc7fa798f7a36955f1b933980401afad2aff592pbos if (voice_engine != nullptr && audio_channel_id != -1) { 3208fc7fa798f7a36955f1b933980401afad2aff592pbos VoEVideoSync* voe_sync_interface = VoEVideoSync::GetInterface(voice_engine); 3218fc7fa798f7a36955f1b933980401afad2aff592pbos vie_channel_->SetVoiceChannel(audio_channel_id, voe_sync_interface); 3228fc7fa798f7a36955f1b933980401afad2aff592pbos voe_sync_interface->Release(); 3238fc7fa798f7a36955f1b933980401afad2aff592pbos } else { 3248fc7fa798f7a36955f1b933980401afad2aff592pbos vie_channel_->SetVoiceChannel(-1, nullptr); 3258fc7fa798f7a36955f1b933980401afad2aff592pbos } 3268fc7fa798f7a36955f1b933980401afad2aff592pbos} 3278fc7fa798f7a36955f1b933980401afad2aff592pbos 3289510e53cc06b0aa5be2be78fbab375216067eea2sprang@webrtc.orgVideoReceiveStream::Stats VideoReceiveStream::GetStats() const { 32909315705b9caf3bff455e3515b9bd99492a7c3e3sprang@webrtc.org return stats_proxy_->GetStats(); 33009315705b9caf3bff455e3515b9bd99492a7c3e3sprang@webrtc.org} 33109315705b9caf3bff455e3515b9bd99492a7c3e3sprang@webrtc.org 332bbb07e69e55761ca43b38f929f3e39f4bba70ef1pbos@webrtc.orgbool VideoReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) { 33376c53d36bc455fe89ca1f860d5171633198fe907Peter Boström return vie_channel_->ReceivedRTCPPacket(packet, length) == 0; 33429d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org} 33529d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 33668786d20400f1f3744ad83549325665c18ea9e5bstefanbool VideoReceiveStream::DeliverRtp(const uint8_t* packet, 33768786d20400f1f3744ad83549325665c18ea9e5bstefan size_t length, 33868786d20400f1f3744ad83549325665c18ea9e5bstefan const PacketTime& packet_time) { 33968786d20400f1f3744ad83549325665c18ea9e5bstefan return vie_channel_->ReceivedRTPPacket(packet, length, packet_time) == 0; 34029d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org} 34129d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 3424765070b8d6f024509c717c04d9b708750666927Miguel Casas-Sanchezvoid VideoReceiveStream::FrameCallback(VideoFrame* video_frame) { 34309315705b9caf3bff455e3515b9bd99492a7c3e3sprang@webrtc.org stats_proxy_->OnDecodedFrame(); 34409315705b9caf3bff455e3515b9bd99492a7c3e3sprang@webrtc.org 3451aff095b6cc39d843a3d85d09de599cfcff0e35eÅsa Persson // Post processing is not supported if the frame is backed by a texture. 3461aff095b6cc39d843a3d85d09de599cfcff0e35eÅsa Persson if (video_frame->native_handle() == NULL) { 3471aff095b6cc39d843a3d85d09de599cfcff0e35eÅsa Persson if (config_.pre_render_callback) 3481aff095b6cc39d843a3d85d09de599cfcff0e35eÅsa Persson config_.pre_render_callback->FrameCallback(video_frame); 3491aff095b6cc39d843a3d85d09de599cfcff0e35eÅsa Persson } 35009315705b9caf3bff455e3515b9bd99492a7c3e3sprang@webrtc.org} 35109315705b9caf3bff455e3515b9bd99492a7c3e3sprang@webrtc.org 352c4188fd3c74688264621393fc622cb81c042c1acPeter Boströmint VideoReceiveStream::RenderFrame(const uint32_t /*stream_id*/, 3534765070b8d6f024509c717c04d9b708750666927Miguel Casas-Sanchez const VideoFrame& video_frame) { 354c4188fd3c74688264621393fc622cb81c042c1acPeter Boström // TODO(pbos): Wire up config_.render->IsTextureSupported() and convert if not 355c4188fd3c74688264621393fc622cb81c042c1acPeter Boström // supported. Or provide methods for converting a texture frame in 3564765070b8d6f024509c717c04d9b708750666927Miguel Casas-Sanchez // VideoFrame. 3570d852d5c27a759fe7aadc500bd7b3cadfae3deb8pbos@webrtc.org 3582b4ce3a501b8d679f84c1ad10317dea5c78fa595pbos@webrtc.org if (config_.renderer != nullptr) 35909315705b9caf3bff455e3515b9bd99492a7c3e3sprang@webrtc.org config_.renderer->RenderFrame( 360f4c10d24dc8f1c3ce6859644077d7df6fb678dcdpbos@webrtc.org video_frame, 361f4c10d24dc8f1c3ce6859644077d7df6fb678dcdpbos@webrtc.org video_frame.render_time_ms() - clock_->TimeInMilliseconds()); 36209315705b9caf3bff455e3515b9bd99492a7c3e3sprang@webrtc.org 363d89920b74a173b7bf80c6760908a382c095a66ccasapersson stats_proxy_->OnRenderedFrame(video_frame.width(), video_frame.height()); 36429d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org 36529d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org return 0; 36629d5839233b19743286a89676bd748a6bd8623d6pbos@webrtc.org} 36726c0c41a06d77af54df547169d952a21319dea8cpbos@webrtc.org 36886b016027d2d27c62fedd108354a2b1274118ae3asapersson// TODO(asapersson): Consider moving callback from video_encoder.h or 36986b016027d2d27c62fedd108354a2b1274118ae3asapersson// creating a different callback. 37086b016027d2d27c62fedd108354a2b1274118ae3asaperssonint32_t VideoReceiveStream::Encoded( 37186b016027d2d27c62fedd108354a2b1274118ae3asapersson const EncodedImage& encoded_image, 37286b016027d2d27c62fedd108354a2b1274118ae3asapersson const CodecSpecificInfo* codec_specific_info, 37386b016027d2d27c62fedd108354a2b1274118ae3asapersson const RTPFragmentationHeader* fragmentation) { 37486b016027d2d27c62fedd108354a2b1274118ae3asapersson stats_proxy_->OnPreDecode(encoded_image, codec_specific_info); 37586b016027d2d27c62fedd108354a2b1274118ae3asapersson if (config_.pre_decode_callback) { 37686b016027d2d27c62fedd108354a2b1274118ae3asapersson // TODO(asapersson): Remove EncodedFrameCallbackAdapter. 37786b016027d2d27c62fedd108354a2b1274118ae3asapersson encoded_frame_proxy_.Encoded( 37886b016027d2d27c62fedd108354a2b1274118ae3asapersson encoded_image, codec_specific_info, fragmentation); 37986b016027d2d27c62fedd108354a2b1274118ae3asapersson } 38086b016027d2d27c62fedd108354a2b1274118ae3asapersson return 0; 38186b016027d2d27c62fedd108354a2b1274118ae3asapersson} 38286b016027d2d27c62fedd108354a2b1274118ae3asapersson 383cd6702282a49448adda470934f4bd9e6181cab22Jelena Marusicvoid VideoReceiveStream::SignalNetworkState(NetworkState state) { 384da903eaabbb6c6830efcafc3c2ade1d36f511e43pbos vie_channel_->SetRTCPMode(state == kNetworkUp ? config_.rtp.rtcp_mode 385da903eaabbb6c6830efcafc3c2ade1d36f511e43pbos : RtcpMode::kOff); 38626c0c41a06d77af54df547169d952a21319dea8cpbos@webrtc.org} 38726c0c41a06d77af54df547169d952a21319dea8cpbos@webrtc.org 388f3973e81d5aa7e4f1d6b5abdfe3a6dc53a32840cmflodman@webrtc.org} // namespace internal 389f3973e81d5aa7e4f1d6b5abdfe3a6dc53a32840cmflodman@webrtc.org} // namespace webrtc 390