webrtcvideoengine2.cc revision 4fbae2b79134572135d9d5fe35a7d1ccdeea3a4d
1/* 2 * libjingle 3 * Copyright 2014 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#ifdef HAVE_WEBRTC_VIDEO 29#include "talk/media/webrtc/webrtcvideoengine2.h" 30 31#include <algorithm> 32#include <set> 33#include <string> 34 35#include "talk/media/base/videocapturer.h" 36#include "talk/media/base/videorenderer.h" 37#include "talk/media/webrtc/constants.h" 38#include "talk/media/webrtc/simulcast.h" 39#include "talk/media/webrtc/webrtcvideoencoderfactory.h" 40#include "talk/media/webrtc/webrtcvideoframe.h" 41#include "talk/media/webrtc/webrtcvoiceengine.h" 42#include "webrtc/base/buffer.h" 43#include "webrtc/base/logging.h" 44#include "webrtc/base/stringutils.h" 45#include "webrtc/base/timeutils.h" 46#include "webrtc/call.h" 47#include "webrtc/modules/video_coding/codecs/h264/include/h264.h" 48#include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h" 49#include "webrtc/system_wrappers/interface/field_trial.h" 50#include "webrtc/system_wrappers/interface/trace_event.h" 51#include "webrtc/video_decoder.h" 52#include "webrtc/video_encoder.h" 53 54#define UNIMPLEMENTED \ 55 LOG(LS_ERROR) << "Call to unimplemented function " << __FUNCTION__; \ 56 RTC_NOTREACHED() 57 58namespace cricket { 59namespace { 60 61// Wrap cricket::WebRtcVideoEncoderFactory as a webrtc::VideoEncoderFactory. 62class EncoderFactoryAdapter : public webrtc::VideoEncoderFactory { 63 public: 64 // EncoderFactoryAdapter doesn't take ownership of |factory|, which is owned 65 // by e.g. PeerConnectionFactory. 66 explicit EncoderFactoryAdapter(cricket::WebRtcVideoEncoderFactory* factory) 67 : factory_(factory) {} 68 virtual ~EncoderFactoryAdapter() {} 69 70 // Implement webrtc::VideoEncoderFactory. 71 webrtc::VideoEncoder* Create() override { 72 return factory_->CreateVideoEncoder(webrtc::kVideoCodecVP8); 73 } 74 75 void Destroy(webrtc::VideoEncoder* encoder) override { 76 return factory_->DestroyVideoEncoder(encoder); 77 } 78 79 private: 80 cricket::WebRtcVideoEncoderFactory* const factory_; 81}; 82 83// An encoder factory that wraps Create requests for simulcastable codec types 84// with a webrtc::SimulcastEncoderAdapter. Non simulcastable codec type 85// requests are just passed through to the contained encoder factory. 86class WebRtcSimulcastEncoderFactory 87 : public cricket::WebRtcVideoEncoderFactory { 88 public: 89 // WebRtcSimulcastEncoderFactory doesn't take ownership of |factory|, which is 90 // owned by e.g. PeerConnectionFactory. 91 explicit WebRtcSimulcastEncoderFactory( 92 cricket::WebRtcVideoEncoderFactory* factory) 93 : factory_(factory) {} 94 95 static bool UseSimulcastEncoderFactory( 96 const std::vector<VideoCodec>& codecs) { 97 // If any codec is VP8, use the simulcast factory. If asked to create a 98 // non-VP8 codec, we'll just return a contained factory encoder directly. 99 for (const auto& codec : codecs) { 100 if (codec.type == webrtc::kVideoCodecVP8) { 101 return true; 102 } 103 } 104 return false; 105 } 106 107 webrtc::VideoEncoder* CreateVideoEncoder( 108 webrtc::VideoCodecType type) override { 109 DCHECK(factory_ != NULL); 110 // If it's a codec type we can simulcast, create a wrapped encoder. 111 if (type == webrtc::kVideoCodecVP8) { 112 return new webrtc::SimulcastEncoderAdapter( 113 new EncoderFactoryAdapter(factory_)); 114 } 115 webrtc::VideoEncoder* encoder = factory_->CreateVideoEncoder(type); 116 if (encoder) { 117 non_simulcast_encoders_.push_back(encoder); 118 } 119 return encoder; 120 } 121 122 const std::vector<VideoCodec>& codecs() const override { 123 return factory_->codecs(); 124 } 125 126 bool EncoderTypeHasInternalSource( 127 webrtc::VideoCodecType type) const override { 128 return factory_->EncoderTypeHasInternalSource(type); 129 } 130 131 void DestroyVideoEncoder(webrtc::VideoEncoder* encoder) override { 132 // Check first to see if the encoder wasn't wrapped in a 133 // SimulcastEncoderAdapter. In that case, ask the factory to destroy it. 134 if (std::remove(non_simulcast_encoders_.begin(), 135 non_simulcast_encoders_.end(), 136 encoder) != non_simulcast_encoders_.end()) { 137 factory_->DestroyVideoEncoder(encoder); 138 return; 139 } 140 141 // Otherwise, SimulcastEncoderAdapter can be deleted directly, and will call 142 // DestroyVideoEncoder on the factory for individual encoder instances. 143 delete encoder; 144 } 145 146 private: 147 cricket::WebRtcVideoEncoderFactory* factory_; 148 // A list of encoders that were created without being wrapped in a 149 // SimulcastEncoderAdapter. 150 std::vector<webrtc::VideoEncoder*> non_simulcast_encoders_; 151}; 152 153bool CodecIsInternallySupported(const std::string& codec_name) { 154 if (CodecNamesEq(codec_name, kVp8CodecName)) { 155 return true; 156 } 157 if (CodecNamesEq(codec_name, kVp9CodecName)) { 158 const std::string group_name = 159 webrtc::field_trial::FindFullName("WebRTC-SupportVP9"); 160 return group_name == "Enabled" || group_name == "EnabledByFlag"; 161 } 162 if (CodecNamesEq(codec_name, kH264CodecName)) { 163 return webrtc::H264Encoder::IsSupported() && 164 webrtc::H264Decoder::IsSupported(); 165 } 166 return false; 167} 168 169void AddDefaultFeedbackParams(VideoCodec* codec) { 170 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamCcm, kRtcpFbCcmParamFir)); 171 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kParamValueEmpty)); 172 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kRtcpFbNackParamPli)); 173 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamRemb, kParamValueEmpty)); 174} 175 176static VideoCodec MakeVideoCodecWithDefaultFeedbackParams(int payload_type, 177 const char* name) { 178 VideoCodec codec(payload_type, name, kDefaultVideoMaxWidth, 179 kDefaultVideoMaxHeight, kDefaultVideoMaxFramerate, 0); 180 AddDefaultFeedbackParams(&codec); 181 return codec; 182} 183 184static std::string CodecVectorToString(const std::vector<VideoCodec>& codecs) { 185 std::stringstream out; 186 out << '{'; 187 for (size_t i = 0; i < codecs.size(); ++i) { 188 out << codecs[i].ToString(); 189 if (i != codecs.size() - 1) { 190 out << ", "; 191 } 192 } 193 out << '}'; 194 return out.str(); 195} 196 197static bool ValidateCodecFormats(const std::vector<VideoCodec>& codecs) { 198 bool has_video = false; 199 for (size_t i = 0; i < codecs.size(); ++i) { 200 if (!codecs[i].ValidateCodecFormat()) { 201 return false; 202 } 203 if (codecs[i].GetCodecType() == VideoCodec::CODEC_VIDEO) { 204 has_video = true; 205 } 206 } 207 if (!has_video) { 208 LOG(LS_ERROR) << "Setting codecs without a video codec is invalid: " 209 << CodecVectorToString(codecs); 210 return false; 211 } 212 return true; 213} 214 215static bool ValidateStreamParams(const StreamParams& sp) { 216 if (sp.ssrcs.empty()) { 217 LOG(LS_ERROR) << "No SSRCs in stream parameters: " << sp.ToString(); 218 return false; 219 } 220 221 std::vector<uint32> primary_ssrcs; 222 sp.GetPrimarySsrcs(&primary_ssrcs); 223 std::vector<uint32> rtx_ssrcs; 224 sp.GetFidSsrcs(primary_ssrcs, &rtx_ssrcs); 225 for (uint32_t rtx_ssrc : rtx_ssrcs) { 226 bool rtx_ssrc_present = false; 227 for (uint32_t sp_ssrc : sp.ssrcs) { 228 if (sp_ssrc == rtx_ssrc) { 229 rtx_ssrc_present = true; 230 break; 231 } 232 } 233 if (!rtx_ssrc_present) { 234 LOG(LS_ERROR) << "RTX SSRC '" << rtx_ssrc 235 << "' missing from StreamParams ssrcs: " << sp.ToString(); 236 return false; 237 } 238 } 239 if (!rtx_ssrcs.empty() && primary_ssrcs.size() != rtx_ssrcs.size()) { 240 LOG(LS_ERROR) 241 << "RTX SSRCs exist, but don't cover all SSRCs (unsupported): " 242 << sp.ToString(); 243 return false; 244 } 245 246 return true; 247} 248 249static std::string RtpExtensionsToString( 250 const std::vector<RtpHeaderExtension>& extensions) { 251 std::stringstream out; 252 out << '{'; 253 for (size_t i = 0; i < extensions.size(); ++i) { 254 out << "{" << extensions[i].uri << ": " << extensions[i].id << "}"; 255 if (i != extensions.size() - 1) { 256 out << ", "; 257 } 258 } 259 out << '}'; 260 return out.str(); 261} 262 263inline const webrtc::RtpExtension* FindHeaderExtension( 264 const std::vector<webrtc::RtpExtension>& extensions, 265 const std::string& name) { 266 for (const auto& kv : extensions) { 267 if (kv.name == name) { 268 return &kv; 269 } 270 } 271 return NULL; 272} 273 274// Merges two fec configs and logs an error if a conflict arises 275// such that merging in different order would trigger a different output. 276static void MergeFecConfig(const webrtc::FecConfig& other, 277 webrtc::FecConfig* output) { 278 if (other.ulpfec_payload_type != -1) { 279 if (output->ulpfec_payload_type != -1 && 280 output->ulpfec_payload_type != other.ulpfec_payload_type) { 281 LOG(LS_WARNING) << "Conflict merging ulpfec_payload_type configs: " 282 << output->ulpfec_payload_type << " and " 283 << other.ulpfec_payload_type; 284 } 285 output->ulpfec_payload_type = other.ulpfec_payload_type; 286 } 287 if (other.red_payload_type != -1) { 288 if (output->red_payload_type != -1 && 289 output->red_payload_type != other.red_payload_type) { 290 LOG(LS_WARNING) << "Conflict merging red_payload_type configs: " 291 << output->red_payload_type << " and " 292 << other.red_payload_type; 293 } 294 output->red_payload_type = other.red_payload_type; 295 } 296 if (other.red_rtx_payload_type != -1) { 297 if (output->red_rtx_payload_type != -1 && 298 output->red_rtx_payload_type != other.red_rtx_payload_type) { 299 LOG(LS_WARNING) << "Conflict merging red_rtx_payload_type configs: " 300 << output->red_rtx_payload_type << " and " 301 << other.red_rtx_payload_type; 302 } 303 output->red_rtx_payload_type = other.red_rtx_payload_type; 304 } 305} 306 307// Returns true if the given codec is disallowed from doing simulcast. 308bool IsCodecBlacklistedForSimulcast(const std::string& codec_name) { 309 return CodecNamesEq(codec_name, kH264CodecName); 310} 311 312} // namespace 313 314// Constants defined in talk/media/webrtc/constants.h 315// TODO(pbos): Move these to a separate constants.cc file. 316const int kMinVideoBitrate = 30; 317const int kStartVideoBitrate = 300; 318const int kMaxVideoBitrate = 2000; 319 320const int kVideoMtu = 1200; 321const int kVideoRtpBufferSize = 65536; 322 323// This constant is really an on/off, lower-level configurable NACK history 324// duration hasn't been implemented. 325static const int kNackHistoryMs = 1000; 326 327static const int kDefaultQpMax = 56; 328 329static const int kDefaultRtcpReceiverReportSsrc = 1; 330 331const int kMinBandwidthBps = 30000; 332const int kStartBandwidthBps = 300000; 333const int kMaxBandwidthBps = 2000000; 334 335std::vector<VideoCodec> DefaultVideoCodecList() { 336 std::vector<VideoCodec> codecs; 337 if (CodecIsInternallySupported(kVp9CodecName)) { 338 codecs.push_back(MakeVideoCodecWithDefaultFeedbackParams(kDefaultVp9PlType, 339 kVp9CodecName)); 340 // TODO(andresp): Add rtx codec for vp9 and verify it works. 341 } 342 codecs.push_back(MakeVideoCodecWithDefaultFeedbackParams(kDefaultVp8PlType, 343 kVp8CodecName)); 344 if (CodecIsInternallySupported(kH264CodecName)) { 345 codecs.push_back(MakeVideoCodecWithDefaultFeedbackParams(kDefaultH264PlType, 346 kH264CodecName)); 347 } 348 codecs.push_back( 349 VideoCodec::CreateRtxCodec(kDefaultRtxVp8PlType, kDefaultVp8PlType)); 350 codecs.push_back(VideoCodec(kDefaultRedPlType, kRedCodecName)); 351 codecs.push_back(VideoCodec(kDefaultUlpfecType, kUlpfecCodecName)); 352 return codecs; 353} 354 355static bool FindFirstMatchingCodec(const std::vector<VideoCodec>& codecs, 356 const VideoCodec& requested_codec, 357 VideoCodec* matching_codec) { 358 for (size_t i = 0; i < codecs.size(); ++i) { 359 if (requested_codec.Matches(codecs[i])) { 360 *matching_codec = codecs[i]; 361 return true; 362 } 363 } 364 return false; 365} 366 367static bool ValidateRtpHeaderExtensionIds( 368 const std::vector<RtpHeaderExtension>& extensions) { 369 std::set<int> extensions_used; 370 for (size_t i = 0; i < extensions.size(); ++i) { 371 if (extensions[i].id <= 0 || extensions[i].id >= 15 || 372 !extensions_used.insert(extensions[i].id).second) { 373 LOG(LS_ERROR) << "RTP extensions are with incorrect or duplicate ids."; 374 return false; 375 } 376 } 377 return true; 378} 379 380static bool CompareRtpHeaderExtensionIds( 381 const webrtc::RtpExtension& extension1, 382 const webrtc::RtpExtension& extension2) { 383 // Sorting on ID is sufficient, more than one extension per ID is unsupported. 384 return extension1.id > extension2.id; 385} 386 387static std::vector<webrtc::RtpExtension> FilterRtpExtensions( 388 const std::vector<RtpHeaderExtension>& extensions) { 389 std::vector<webrtc::RtpExtension> webrtc_extensions; 390 for (size_t i = 0; i < extensions.size(); ++i) { 391 // Unsupported extensions will be ignored. 392 if (webrtc::RtpExtension::IsSupportedForVideo(extensions[i].uri)) { 393 webrtc_extensions.push_back(webrtc::RtpExtension( 394 extensions[i].uri, extensions[i].id)); 395 } else { 396 LOG(LS_WARNING) << "Unsupported RTP extension: " << extensions[i].uri; 397 } 398 } 399 400 // Sort filtered headers to make sure that they can later be compared 401 // regardless of in which order they were entered. 402 std::sort(webrtc_extensions.begin(), webrtc_extensions.end(), 403 CompareRtpHeaderExtensionIds); 404 return webrtc_extensions; 405} 406 407static bool RtpExtensionsHaveChanged( 408 const std::vector<webrtc::RtpExtension>& before, 409 const std::vector<webrtc::RtpExtension>& after) { 410 if (before.size() != after.size()) 411 return true; 412 for (size_t i = 0; i < before.size(); ++i) { 413 if (before[i].id != after[i].id) 414 return true; 415 if (before[i].name != after[i].name) 416 return true; 417 } 418 return false; 419} 420 421std::vector<webrtc::VideoStream> 422WebRtcVideoChannel2::WebRtcVideoSendStream::CreateSimulcastVideoStreams( 423 const VideoCodec& codec, 424 const VideoOptions& options, 425 int max_bitrate_bps, 426 size_t num_streams) { 427 int max_qp = kDefaultQpMax; 428 codec.GetParam(kCodecParamMaxQuantization, &max_qp); 429 430 return GetSimulcastConfig( 431 num_streams, GetSimulcastBitrateMode(options), codec.width, codec.height, 432 max_bitrate_bps, max_qp, 433 codec.framerate != 0 ? codec.framerate : kDefaultVideoMaxFramerate); 434} 435 436std::vector<webrtc::VideoStream> 437WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoStreams( 438 const VideoCodec& codec, 439 const VideoOptions& options, 440 int max_bitrate_bps, 441 size_t num_streams) { 442 int codec_max_bitrate_kbps; 443 if (codec.GetParam(kCodecParamMaxBitrate, &codec_max_bitrate_kbps)) { 444 max_bitrate_bps = codec_max_bitrate_kbps * 1000; 445 } 446 if (num_streams != 1) { 447 return CreateSimulcastVideoStreams(codec, options, max_bitrate_bps, 448 num_streams); 449 } 450 451 // For unset max bitrates set default bitrate for non-simulcast. 452 if (max_bitrate_bps <= 0) 453 max_bitrate_bps = kMaxVideoBitrate * 1000; 454 455 webrtc::VideoStream stream; 456 stream.width = codec.width; 457 stream.height = codec.height; 458 stream.max_framerate = 459 codec.framerate != 0 ? codec.framerate : kDefaultVideoMaxFramerate; 460 461 stream.min_bitrate_bps = kMinVideoBitrate * 1000; 462 stream.target_bitrate_bps = stream.max_bitrate_bps = max_bitrate_bps; 463 464 int max_qp = kDefaultQpMax; 465 codec.GetParam(kCodecParamMaxQuantization, &max_qp); 466 stream.max_qp = max_qp; 467 std::vector<webrtc::VideoStream> streams; 468 streams.push_back(stream); 469 return streams; 470} 471 472void* WebRtcVideoChannel2::WebRtcVideoSendStream::ConfigureVideoEncoderSettings( 473 const VideoCodec& codec, 474 const VideoOptions& options, 475 bool is_screencast) { 476 // No automatic resizing when using simulcast. 477 bool automatic_resize = !is_screencast && ssrcs_.size() == 1; 478 bool frame_dropping = !is_screencast; 479 bool denoising; 480 if (is_screencast) { 481 denoising = false; 482 } else { 483 options.video_noise_reduction.Get(&denoising); 484 } 485 486 if (CodecNamesEq(codec.name, kVp8CodecName)) { 487 encoder_settings_.vp8 = webrtc::VideoEncoder::GetDefaultVp8Settings(); 488 encoder_settings_.vp8.automaticResizeOn = automatic_resize; 489 encoder_settings_.vp8.denoisingOn = denoising; 490 encoder_settings_.vp8.frameDroppingOn = frame_dropping; 491 return &encoder_settings_.vp8; 492 } 493 if (CodecNamesEq(codec.name, kVp9CodecName)) { 494 encoder_settings_.vp9 = webrtc::VideoEncoder::GetDefaultVp9Settings(); 495 encoder_settings_.vp9.denoisingOn = denoising; 496 encoder_settings_.vp9.frameDroppingOn = frame_dropping; 497 return &encoder_settings_.vp9; 498 } 499 return NULL; 500} 501 502DefaultUnsignalledSsrcHandler::DefaultUnsignalledSsrcHandler() 503 : default_recv_ssrc_(0), default_renderer_(NULL) {} 504 505UnsignalledSsrcHandler::Action DefaultUnsignalledSsrcHandler::OnUnsignalledSsrc( 506 WebRtcVideoChannel2* channel, 507 uint32_t ssrc) { 508 if (default_recv_ssrc_ != 0) { // Already one default stream. 509 LOG(LS_WARNING) << "Unknown SSRC, but default receive stream already set."; 510 return kDropPacket; 511 } 512 513 StreamParams sp; 514 sp.ssrcs.push_back(ssrc); 515 LOG(LS_INFO) << "Creating default receive stream for SSRC=" << ssrc << "."; 516 if (!channel->AddRecvStream(sp, true)) { 517 LOG(LS_WARNING) << "Could not create default receive stream."; 518 } 519 520 channel->SetRenderer(ssrc, default_renderer_); 521 default_recv_ssrc_ = ssrc; 522 return kDeliverPacket; 523} 524 525WebRtcCallFactory::~WebRtcCallFactory() { 526} 527webrtc::Call* WebRtcCallFactory::CreateCall( 528 const webrtc::Call::Config& config) { 529 return webrtc::Call::Create(config); 530} 531 532VideoRenderer* DefaultUnsignalledSsrcHandler::GetDefaultRenderer() const { 533 return default_renderer_; 534} 535 536void DefaultUnsignalledSsrcHandler::SetDefaultRenderer( 537 VideoMediaChannel* channel, 538 VideoRenderer* renderer) { 539 default_renderer_ = renderer; 540 if (default_recv_ssrc_ != 0) { 541 channel->SetRenderer(default_recv_ssrc_, default_renderer_); 542 } 543} 544 545WebRtcVideoEngine2::WebRtcVideoEngine2(WebRtcVoiceEngine* voice_engine) 546 : voice_engine_(voice_engine), 547 initialized_(false), 548 call_factory_(&default_call_factory_), 549 external_decoder_factory_(NULL), 550 external_encoder_factory_(NULL) { 551 LOG(LS_INFO) << "WebRtcVideoEngine2::WebRtcVideoEngine2()"; 552 video_codecs_ = GetSupportedCodecs(); 553 rtp_header_extensions_.push_back( 554 RtpHeaderExtension(kRtpTimestampOffsetHeaderExtension, 555 kRtpTimestampOffsetHeaderExtensionDefaultId)); 556 rtp_header_extensions_.push_back( 557 RtpHeaderExtension(kRtpAbsoluteSenderTimeHeaderExtension, 558 kRtpAbsoluteSenderTimeHeaderExtensionDefaultId)); 559 rtp_header_extensions_.push_back( 560 RtpHeaderExtension(kRtpVideoRotationHeaderExtension, 561 kRtpVideoRotationHeaderExtensionDefaultId)); 562} 563 564WebRtcVideoEngine2::~WebRtcVideoEngine2() { 565 LOG(LS_INFO) << "WebRtcVideoEngine2::~WebRtcVideoEngine2"; 566} 567 568void WebRtcVideoEngine2::SetCallFactory(WebRtcCallFactory* call_factory) { 569 DCHECK(!initialized_); 570 call_factory_ = call_factory; 571} 572 573void WebRtcVideoEngine2::Init() { 574 LOG(LS_INFO) << "WebRtcVideoEngine2::Init"; 575 initialized_ = true; 576} 577 578int WebRtcVideoEngine2::GetCapabilities() { return VIDEO_RECV | VIDEO_SEND; } 579 580bool WebRtcVideoEngine2::SetDefaultEncoderConfig( 581 const VideoEncoderConfig& config) { 582 const VideoCodec& codec = config.max_codec; 583 bool supports_codec = false; 584 for (size_t i = 0; i < video_codecs_.size(); ++i) { 585 if (CodecNamesEq(video_codecs_[i].name, codec.name)) { 586 video_codecs_[i].width = codec.width; 587 video_codecs_[i].height = codec.height; 588 video_codecs_[i].framerate = codec.framerate; 589 supports_codec = true; 590 break; 591 } 592 } 593 594 if (!supports_codec) { 595 LOG(LS_ERROR) << "SetDefaultEncoderConfig, codec not supported: " 596 << codec.ToString(); 597 return false; 598 } 599 600 return true; 601} 602 603WebRtcVideoChannel2* WebRtcVideoEngine2::CreateChannel( 604 const VideoOptions& options, 605 VoiceMediaChannel* voice_channel) { 606 DCHECK(initialized_); 607 LOG(LS_INFO) << "CreateChannel: " 608 << (voice_channel != NULL ? "With" : "Without") 609 << " voice channel. Options: " << options.ToString(); 610 WebRtcVideoChannel2* channel = 611 new WebRtcVideoChannel2(call_factory_, voice_engine_, 612 static_cast<WebRtcVoiceMediaChannel*>(voice_channel), options, 613 external_encoder_factory_, external_decoder_factory_); 614 if (!channel->Init()) { 615 delete channel; 616 return NULL; 617 } 618 channel->SetRecvCodecs(video_codecs_); 619 return channel; 620} 621 622const std::vector<VideoCodec>& WebRtcVideoEngine2::codecs() const { 623 return video_codecs_; 624} 625 626const std::vector<RtpHeaderExtension>& 627WebRtcVideoEngine2::rtp_header_extensions() const { 628 return rtp_header_extensions_; 629} 630 631void WebRtcVideoEngine2::SetLogging(int min_sev, const char* filter) { 632 // TODO(pbos): Set up logging. 633 LOG(LS_VERBOSE) << "SetLogging: " << min_sev << '"' << filter << '"'; 634 // if min_sev == -1, we keep the current log level. 635 if (min_sev < 0) { 636 DCHECK(min_sev == -1); 637 return; 638 } 639} 640 641void WebRtcVideoEngine2::SetExternalDecoderFactory( 642 WebRtcVideoDecoderFactory* decoder_factory) { 643 DCHECK(!initialized_); 644 external_decoder_factory_ = decoder_factory; 645} 646 647void WebRtcVideoEngine2::SetExternalEncoderFactory( 648 WebRtcVideoEncoderFactory* encoder_factory) { 649 DCHECK(!initialized_); 650 if (external_encoder_factory_ == encoder_factory) 651 return; 652 653 // No matter what happens we shouldn't hold on to a stale 654 // WebRtcSimulcastEncoderFactory. 655 simulcast_encoder_factory_.reset(); 656 657 if (encoder_factory && 658 WebRtcSimulcastEncoderFactory::UseSimulcastEncoderFactory( 659 encoder_factory->codecs())) { 660 simulcast_encoder_factory_.reset( 661 new WebRtcSimulcastEncoderFactory(encoder_factory)); 662 encoder_factory = simulcast_encoder_factory_.get(); 663 } 664 external_encoder_factory_ = encoder_factory; 665 666 video_codecs_ = GetSupportedCodecs(); 667} 668 669bool WebRtcVideoEngine2::EnableTimedRender() { 670 // TODO(pbos): Figure out whether this can be removed. 671 return true; 672} 673 674// Checks to see whether we comprehend and could receive a particular codec 675bool WebRtcVideoEngine2::FindCodec(const VideoCodec& in) { 676 // TODO(pbos): Probe encoder factory to figure out that the codec is supported 677 // if supported by the encoder factory. Add a corresponding test that fails 678 // with this code (that doesn't ask the factory). 679 for (size_t j = 0; j < video_codecs_.size(); ++j) { 680 VideoCodec codec(video_codecs_[j].id, video_codecs_[j].name, 0, 0, 0, 0); 681 if (codec.Matches(in)) { 682 return true; 683 } 684 } 685 return false; 686} 687 688// Tells whether the |requested| codec can be transmitted or not. If it can be 689// transmitted |out| is set with the best settings supported. Aspect ratio will 690// be set as close to |current|'s as possible. If not set |requested|'s 691// dimensions will be used for aspect ratio matching. 692bool WebRtcVideoEngine2::CanSendCodec(const VideoCodec& requested, 693 const VideoCodec& current, 694 VideoCodec* out) { 695 DCHECK(out != NULL); 696 697 if (requested.width != requested.height && 698 (requested.height == 0 || requested.width == 0)) { 699 // 0xn and nx0 are invalid resolutions. 700 return false; 701 } 702 703 VideoCodec matching_codec; 704 if (!FindFirstMatchingCodec(video_codecs_, requested, &matching_codec)) { 705 // Codec not supported. 706 return false; 707 } 708 709 out->id = requested.id; 710 out->name = requested.name; 711 out->preference = requested.preference; 712 out->params = requested.params; 713 out->framerate = std::min(requested.framerate, matching_codec.framerate); 714 out->params = requested.params; 715 out->feedback_params = requested.feedback_params; 716 out->width = requested.width; 717 out->height = requested.height; 718 if (requested.width == 0 && requested.height == 0) { 719 return true; 720 } 721 722 while (out->width > matching_codec.width) { 723 out->width /= 2; 724 out->height /= 2; 725 } 726 727 return out->width > 0 && out->height > 0; 728} 729 730// Ignore spammy trace messages, mostly from the stats API when we haven't 731// gotten RTCP info yet from the remote side. 732bool WebRtcVideoEngine2::ShouldIgnoreTrace(const std::string& trace) { 733 static const char* const kTracesToIgnore[] = {NULL}; 734 for (const char* const* p = kTracesToIgnore; *p; ++p) { 735 if (trace.find(*p) == 0) { 736 return true; 737 } 738 } 739 return false; 740} 741 742std::vector<VideoCodec> WebRtcVideoEngine2::GetSupportedCodecs() const { 743 std::vector<VideoCodec> supported_codecs = DefaultVideoCodecList(); 744 745 if (external_encoder_factory_ == NULL) { 746 return supported_codecs; 747 } 748 749 const std::vector<WebRtcVideoEncoderFactory::VideoCodec>& codecs = 750 external_encoder_factory_->codecs(); 751 for (size_t i = 0; i < codecs.size(); ++i) { 752 // Don't add internally-supported codecs twice. 753 if (CodecIsInternallySupported(codecs[i].name)) { 754 continue; 755 } 756 757 // External video encoders are given payloads 120-127. This also means that 758 // we only support up to 8 external payload types. 759 const int kExternalVideoPayloadTypeBase = 120; 760 size_t payload_type = kExternalVideoPayloadTypeBase + i; 761 DCHECK(payload_type < 128); 762 VideoCodec codec(static_cast<int>(payload_type), 763 codecs[i].name, 764 codecs[i].max_width, 765 codecs[i].max_height, 766 codecs[i].max_fps, 767 0); 768 769 AddDefaultFeedbackParams(&codec); 770 supported_codecs.push_back(codec); 771 } 772 return supported_codecs; 773} 774 775WebRtcVideoChannel2::WebRtcVideoChannel2( 776 WebRtcCallFactory* call_factory, 777 WebRtcVoiceEngine* voice_engine, 778 WebRtcVoiceMediaChannel* voice_channel, 779 const VideoOptions& options, 780 WebRtcVideoEncoderFactory* external_encoder_factory, 781 WebRtcVideoDecoderFactory* external_decoder_factory) 782 : unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_), 783 voice_channel_(voice_channel), 784 voice_channel_id_(voice_channel ? voice_channel->voe_channel() : -1), 785 external_encoder_factory_(external_encoder_factory), 786 external_decoder_factory_(external_decoder_factory) { 787 DCHECK(thread_checker_.CalledOnValidThread()); 788 SetDefaultOptions(); 789 options_.SetAll(options); 790 options_.cpu_overuse_detection.Get(&signal_cpu_adaptation_); 791 webrtc::Call::Config config; 792 config.overuse_callback = this; 793 if (voice_engine != NULL) { 794 config.voice_engine = voice_engine->voe()->engine(); 795 } 796 config.bitrate_config.min_bitrate_bps = kMinBandwidthBps; 797 config.bitrate_config.start_bitrate_bps = kStartBandwidthBps; 798 config.bitrate_config.max_bitrate_bps = kMaxBandwidthBps; 799 call_.reset(call_factory->CreateCall(config)); 800 if (voice_channel_) { 801 voice_channel_->SetCall(call_.get()); 802 } 803 rtcp_receiver_report_ssrc_ = kDefaultRtcpReceiverReportSsrc; 804 sending_ = false; 805 default_send_ssrc_ = 0; 806} 807 808void WebRtcVideoChannel2::SetDefaultOptions() { 809 options_.cpu_overuse_detection.Set(true); 810 options_.dscp.Set(false); 811 options_.suspend_below_min_bitrate.Set(false); 812 options_.video_noise_reduction.Set(true); 813 options_.screencast_min_bitrate.Set(0); 814} 815 816WebRtcVideoChannel2::~WebRtcVideoChannel2() { 817 DetachVoiceChannel(); 818 for (auto& kv : send_streams_) 819 delete kv.second; 820 for (auto& kv : receive_streams_) 821 delete kv.second; 822} 823 824bool WebRtcVideoChannel2::Init() { return true; } 825 826void WebRtcVideoChannel2::DetachVoiceChannel() { 827 DCHECK(thread_checker_.CalledOnValidThread()); 828 if (voice_channel_) { 829 voice_channel_->SetCall(nullptr); 830 voice_channel_ = nullptr; 831 } 832} 833 834bool WebRtcVideoChannel2::CodecIsExternallySupported( 835 const std::string& name) const { 836 if (external_encoder_factory_ == NULL) { 837 return false; 838 } 839 840 const std::vector<WebRtcVideoEncoderFactory::VideoCodec> external_codecs = 841 external_encoder_factory_->codecs(); 842 for (size_t c = 0; c < external_codecs.size(); ++c) { 843 if (CodecNamesEq(name, external_codecs[c].name)) { 844 return true; 845 } 846 } 847 return false; 848} 849 850std::vector<WebRtcVideoChannel2::VideoCodecSettings> 851WebRtcVideoChannel2::FilterSupportedCodecs( 852 const std::vector<WebRtcVideoChannel2::VideoCodecSettings>& mapped_codecs) 853 const { 854 std::vector<VideoCodecSettings> supported_codecs; 855 for (size_t i = 0; i < mapped_codecs.size(); ++i) { 856 const VideoCodecSettings& codec = mapped_codecs[i]; 857 if (CodecIsInternallySupported(codec.codec.name) || 858 CodecIsExternallySupported(codec.codec.name)) { 859 supported_codecs.push_back(codec); 860 } 861 } 862 return supported_codecs; 863} 864 865bool WebRtcVideoChannel2::ReceiveCodecsHaveChanged( 866 std::vector<VideoCodecSettings> before, 867 std::vector<VideoCodecSettings> after) { 868 if (before.size() != after.size()) { 869 return true; 870 } 871 // The receive codec order doesn't matter, so we sort the codecs before 872 // comparing. This is necessary because currently the 873 // only way to change the send codec is to munge SDP, which causes 874 // the receive codec list to change order, which causes the streams 875 // to be recreates which causes a "blink" of black video. In order 876 // to support munging the SDP in this way without recreating receive 877 // streams, we ignore the order of the received codecs so that 878 // changing the order doesn't cause this "blink". 879 auto comparison = 880 [](const VideoCodecSettings& codec1, const VideoCodecSettings& codec2) { 881 return codec1.codec.id > codec2.codec.id; 882 }; 883 std::sort(before.begin(), before.end(), comparison); 884 std::sort(after.begin(), after.end(), comparison); 885 for (size_t i = 0; i < before.size(); ++i) { 886 // For the same reason that we sort the codecs, we also ignore the 887 // preference. We don't want a preference change on the receive 888 // side to cause recreation of the stream. 889 before[i].codec.preference = 0; 890 after[i].codec.preference = 0; 891 if (before[i] != after[i]) { 892 return true; 893 } 894 } 895 return false; 896} 897 898bool WebRtcVideoChannel2::SetSendParameters(const VideoSendParameters& params) { 899 // TODO(pbos): Refactor this to only recreate the send streams once 900 // instead of 4 times. 901 return (SetSendCodecs(params.codecs) && 902 SetSendRtpHeaderExtensions(params.extensions) && 903 SetMaxSendBandwidth(params.max_bandwidth_bps) && 904 SetOptions(params.options)); 905} 906 907bool WebRtcVideoChannel2::SetRecvParameters(const VideoRecvParameters& params) { 908 // TODO(pbos): Refactor this to only recreate the recv streams once 909 // instead of twice. 910 return (SetRecvCodecs(params.codecs) && 911 SetRecvRtpHeaderExtensions(params.extensions)); 912} 913 914std::string WebRtcVideoChannel2::CodecSettingsVectorToString( 915 const std::vector<VideoCodecSettings>& codecs) { 916 std::stringstream out; 917 out << '{'; 918 for (size_t i = 0; i < codecs.size(); ++i) { 919 out << codecs[i].codec.ToString(); 920 if (i != codecs.size() - 1) { 921 out << ", "; 922 } 923 } 924 out << '}'; 925 return out.str(); 926} 927 928bool WebRtcVideoChannel2::SetRecvCodecs(const std::vector<VideoCodec>& codecs) { 929 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetRecvCodecs"); 930 LOG(LS_INFO) << "SetRecvCodecs: " << CodecVectorToString(codecs); 931 if (!ValidateCodecFormats(codecs)) { 932 return false; 933 } 934 935 const std::vector<VideoCodecSettings> mapped_codecs = MapCodecs(codecs); 936 if (mapped_codecs.empty()) { 937 LOG(LS_ERROR) << "SetRecvCodecs called without any video codecs."; 938 return false; 939 } 940 941 std::vector<VideoCodecSettings> supported_codecs = 942 FilterSupportedCodecs(mapped_codecs); 943 944 if (mapped_codecs.size() != supported_codecs.size()) { 945 LOG(LS_ERROR) << "SetRecvCodecs called with unsupported video codecs."; 946 return false; 947 } 948 949 // Prevent reconfiguration when setting identical receive codecs. 950 if (!ReceiveCodecsHaveChanged(recv_codecs_, supported_codecs)) { 951 LOG(LS_INFO) 952 << "Ignoring call to SetRecvCodecs because codecs haven't changed."; 953 return true; 954 } 955 956 LOG(LS_INFO) << "Changing recv codecs from " 957 << CodecSettingsVectorToString(recv_codecs_) << " to " 958 << CodecSettingsVectorToString(supported_codecs); 959 recv_codecs_ = supported_codecs; 960 961 rtc::CritScope stream_lock(&stream_crit_); 962 for (std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 963 receive_streams_.begin(); 964 it != receive_streams_.end(); 965 ++it) { 966 it->second->SetRecvCodecs(recv_codecs_); 967 } 968 969 return true; 970} 971 972bool WebRtcVideoChannel2::SetSendCodecs(const std::vector<VideoCodec>& codecs) { 973 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetSendCodecs"); 974 LOG(LS_INFO) << "SetSendCodecs: " << CodecVectorToString(codecs); 975 if (!ValidateCodecFormats(codecs)) { 976 return false; 977 } 978 979 const std::vector<VideoCodecSettings> supported_codecs = 980 FilterSupportedCodecs(MapCodecs(codecs)); 981 982 if (supported_codecs.empty()) { 983 LOG(LS_ERROR) << "No video codecs supported."; 984 return false; 985 } 986 987 LOG(LS_INFO) << "Using codec: " << supported_codecs.front().codec.ToString(); 988 989 VideoCodecSettings old_codec; 990 if (send_codec_.Get(&old_codec) && supported_codecs.front() == old_codec) { 991 LOG(LS_INFO) << "Ignore call to SetSendCodecs because first supported " 992 "codec hasn't changed."; 993 // Using same codec, avoid reconfiguring. 994 return true; 995 } 996 997 send_codec_.Set(supported_codecs.front()); 998 999 rtc::CritScope stream_lock(&stream_crit_); 1000 LOG(LS_INFO) << "Change the send codec because SetSendCodecs has a different " 1001 "first supported codec."; 1002 for (auto& kv : send_streams_) { 1003 DCHECK(kv.second != nullptr); 1004 kv.second->SetCodec(supported_codecs.front()); 1005 } 1006 LOG(LS_INFO) << "SetNackAndRemb on all the receive streams because the send " 1007 "codec has changed."; 1008 for (auto& kv : receive_streams_) { 1009 DCHECK(kv.second != nullptr); 1010 kv.second->SetNackAndRemb(HasNack(supported_codecs.front().codec), 1011 HasRemb(supported_codecs.front().codec)); 1012 } 1013 1014 // TODO(holmer): Changing the codec parameters shouldn't necessarily mean that 1015 // we change the min/max of bandwidth estimation. Reevaluate this. 1016 VideoCodec codec = supported_codecs.front().codec; 1017 int bitrate_kbps; 1018 if (codec.GetParam(kCodecParamMinBitrate, &bitrate_kbps) && 1019 bitrate_kbps > 0) { 1020 bitrate_config_.min_bitrate_bps = bitrate_kbps * 1000; 1021 } else { 1022 bitrate_config_.min_bitrate_bps = 0; 1023 } 1024 if (codec.GetParam(kCodecParamStartBitrate, &bitrate_kbps) && 1025 bitrate_kbps > 0) { 1026 bitrate_config_.start_bitrate_bps = bitrate_kbps * 1000; 1027 } else { 1028 // Do not reconfigure start bitrate unless it's specified and positive. 1029 bitrate_config_.start_bitrate_bps = -1; 1030 } 1031 if (codec.GetParam(kCodecParamMaxBitrate, &bitrate_kbps) && 1032 bitrate_kbps > 0) { 1033 bitrate_config_.max_bitrate_bps = bitrate_kbps * 1000; 1034 } else { 1035 bitrate_config_.max_bitrate_bps = -1; 1036 } 1037 call_->SetBitrateConfig(bitrate_config_); 1038 1039 return true; 1040} 1041 1042bool WebRtcVideoChannel2::GetSendCodec(VideoCodec* codec) { 1043 VideoCodecSettings codec_settings; 1044 if (!send_codec_.Get(&codec_settings)) { 1045 LOG(LS_VERBOSE) << "GetSendCodec: No send codec set."; 1046 return false; 1047 } 1048 *codec = codec_settings.codec; 1049 return true; 1050} 1051 1052bool WebRtcVideoChannel2::SetSendStreamFormat(uint32 ssrc, 1053 const VideoFormat& format) { 1054 LOG(LS_VERBOSE) << "SetSendStreamFormat:" << ssrc << " -> " 1055 << format.ToString(); 1056 rtc::CritScope stream_lock(&stream_crit_); 1057 if (send_streams_.find(ssrc) == send_streams_.end()) { 1058 return false; 1059 } 1060 return send_streams_[ssrc]->SetVideoFormat(format); 1061} 1062 1063bool WebRtcVideoChannel2::SetRender(bool render) { 1064 // TODO(pbos): Implement. Or refactor away as it shouldn't be needed. 1065 LOG(LS_VERBOSE) << "SetRender: " << (render ? "true" : "false"); 1066 return true; 1067} 1068 1069bool WebRtcVideoChannel2::SetSend(bool send) { 1070 LOG(LS_VERBOSE) << "SetSend: " << (send ? "true" : "false"); 1071 if (send && !send_codec_.IsSet()) { 1072 LOG(LS_ERROR) << "SetSend(true) called before setting codec."; 1073 return false; 1074 } 1075 if (send) { 1076 StartAllSendStreams(); 1077 } else { 1078 StopAllSendStreams(); 1079 } 1080 sending_ = send; 1081 return true; 1082} 1083 1084bool WebRtcVideoChannel2::ValidateSendSsrcAvailability( 1085 const StreamParams& sp) const { 1086 for (uint32_t ssrc: sp.ssrcs) { 1087 if (send_ssrcs_.find(ssrc) != send_ssrcs_.end()) { 1088 LOG(LS_ERROR) << "Send stream with SSRC '" << ssrc << "' already exists."; 1089 return false; 1090 } 1091 } 1092 return true; 1093} 1094 1095bool WebRtcVideoChannel2::ValidateReceiveSsrcAvailability( 1096 const StreamParams& sp) const { 1097 for (uint32_t ssrc: sp.ssrcs) { 1098 if (receive_ssrcs_.find(ssrc) != receive_ssrcs_.end()) { 1099 LOG(LS_ERROR) << "Receive stream with SSRC '" << ssrc 1100 << "' already exists."; 1101 return false; 1102 } 1103 } 1104 return true; 1105} 1106 1107bool WebRtcVideoChannel2::AddSendStream(const StreamParams& sp) { 1108 LOG(LS_INFO) << "AddSendStream: " << sp.ToString(); 1109 if (!ValidateStreamParams(sp)) 1110 return false; 1111 1112 rtc::CritScope stream_lock(&stream_crit_); 1113 1114 if (!ValidateSendSsrcAvailability(sp)) 1115 return false; 1116 1117 for (uint32 used_ssrc : sp.ssrcs) 1118 send_ssrcs_.insert(used_ssrc); 1119 1120 WebRtcVideoSendStream* stream = 1121 new WebRtcVideoSendStream(call_.get(), 1122 sp, 1123 webrtc::VideoSendStream::Config(this), 1124 external_encoder_factory_, 1125 options_, 1126 bitrate_config_.max_bitrate_bps, 1127 send_codec_, 1128 send_rtp_extensions_); 1129 1130 uint32 ssrc = sp.first_ssrc(); 1131 DCHECK(ssrc != 0); 1132 send_streams_[ssrc] = stream; 1133 1134 if (rtcp_receiver_report_ssrc_ == kDefaultRtcpReceiverReportSsrc) { 1135 rtcp_receiver_report_ssrc_ = ssrc; 1136 LOG(LS_INFO) << "SetLocalSsrc on all the receive streams because we added " 1137 "a send stream."; 1138 for (auto& kv : receive_streams_) 1139 kv.second->SetLocalSsrc(ssrc); 1140 } 1141 if (default_send_ssrc_ == 0) { 1142 default_send_ssrc_ = ssrc; 1143 } 1144 if (sending_) { 1145 stream->Start(); 1146 } 1147 1148 return true; 1149} 1150 1151bool WebRtcVideoChannel2::RemoveSendStream(uint32 ssrc) { 1152 LOG(LS_INFO) << "RemoveSendStream: " << ssrc; 1153 1154 if (ssrc == 0) { 1155 if (default_send_ssrc_ == 0) { 1156 LOG(LS_ERROR) << "No default send stream active."; 1157 return false; 1158 } 1159 1160 LOG(LS_VERBOSE) << "Removing default stream: " << default_send_ssrc_; 1161 ssrc = default_send_ssrc_; 1162 } 1163 1164 WebRtcVideoSendStream* removed_stream; 1165 { 1166 rtc::CritScope stream_lock(&stream_crit_); 1167 std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1168 send_streams_.find(ssrc); 1169 if (it == send_streams_.end()) { 1170 return false; 1171 } 1172 1173 for (uint32 old_ssrc : it->second->GetSsrcs()) 1174 send_ssrcs_.erase(old_ssrc); 1175 1176 removed_stream = it->second; 1177 send_streams_.erase(it); 1178 } 1179 1180 delete removed_stream; 1181 1182 if (ssrc == default_send_ssrc_) { 1183 default_send_ssrc_ = 0; 1184 } 1185 1186 return true; 1187} 1188 1189void WebRtcVideoChannel2::DeleteReceiveStream( 1190 WebRtcVideoChannel2::WebRtcVideoReceiveStream* stream) { 1191 for (uint32 old_ssrc : stream->GetSsrcs()) 1192 receive_ssrcs_.erase(old_ssrc); 1193 delete stream; 1194} 1195 1196bool WebRtcVideoChannel2::AddRecvStream(const StreamParams& sp) { 1197 return AddRecvStream(sp, false); 1198} 1199 1200bool WebRtcVideoChannel2::AddRecvStream(const StreamParams& sp, 1201 bool default_stream) { 1202 DCHECK(thread_checker_.CalledOnValidThread()); 1203 1204 LOG(LS_INFO) << "AddRecvStream" << (default_stream ? " (default stream)" : "") 1205 << ": " << sp.ToString(); 1206 if (!ValidateStreamParams(sp)) 1207 return false; 1208 1209 uint32 ssrc = sp.first_ssrc(); 1210 DCHECK(ssrc != 0); // TODO(pbos): Is this ever valid? 1211 1212 rtc::CritScope stream_lock(&stream_crit_); 1213 // Remove running stream if this was a default stream. 1214 auto prev_stream = receive_streams_.find(ssrc); 1215 if (prev_stream != receive_streams_.end()) { 1216 if (default_stream || !prev_stream->second->IsDefaultStream()) { 1217 LOG(LS_ERROR) << "Receive stream for SSRC '" << ssrc 1218 << "' already exists."; 1219 return false; 1220 } 1221 DeleteReceiveStream(prev_stream->second); 1222 receive_streams_.erase(prev_stream); 1223 } 1224 1225 if (!ValidateReceiveSsrcAvailability(sp)) 1226 return false; 1227 1228 for (uint32 used_ssrc : sp.ssrcs) 1229 receive_ssrcs_.insert(used_ssrc); 1230 1231 webrtc::VideoReceiveStream::Config config(this); 1232 ConfigureReceiverRtp(&config, sp); 1233 1234 // Set up A/V sync group based on sync label. 1235 config.sync_group = sp.sync_label; 1236 1237 config.rtp.remb = false; 1238 VideoCodecSettings send_codec; 1239 if (send_codec_.Get(&send_codec)) { 1240 config.rtp.remb = HasRemb(send_codec.codec); 1241 } 1242 1243 receive_streams_[ssrc] = new WebRtcVideoReceiveStream( 1244 call_.get(), sp, config, external_decoder_factory_, default_stream, 1245 recv_codecs_); 1246 1247 return true; 1248} 1249 1250void WebRtcVideoChannel2::ConfigureReceiverRtp( 1251 webrtc::VideoReceiveStream::Config* config, 1252 const StreamParams& sp) const { 1253 uint32 ssrc = sp.first_ssrc(); 1254 1255 config->rtp.remote_ssrc = ssrc; 1256 config->rtp.local_ssrc = rtcp_receiver_report_ssrc_; 1257 1258 config->rtp.extensions = recv_rtp_extensions_; 1259 1260 // TODO(pbos): This protection is against setting the same local ssrc as 1261 // remote which is not permitted by the lower-level API. RTCP requires a 1262 // corresponding sender SSRC. Figure out what to do when we don't have 1263 // (receive-only) or know a good local SSRC. 1264 if (config->rtp.remote_ssrc == config->rtp.local_ssrc) { 1265 if (config->rtp.local_ssrc != kDefaultRtcpReceiverReportSsrc) { 1266 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc; 1267 } else { 1268 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc + 1; 1269 } 1270 } 1271 1272 for (size_t i = 0; i < recv_codecs_.size(); ++i) { 1273 MergeFecConfig(recv_codecs_[i].fec, &config->rtp.fec); 1274 } 1275 1276 for (size_t i = 0; i < recv_codecs_.size(); ++i) { 1277 uint32 rtx_ssrc; 1278 if (recv_codecs_[i].rtx_payload_type != -1 && 1279 sp.GetFidSsrc(ssrc, &rtx_ssrc)) { 1280 webrtc::VideoReceiveStream::Config::Rtp::Rtx& rtx = 1281 config->rtp.rtx[recv_codecs_[i].codec.id]; 1282 rtx.ssrc = rtx_ssrc; 1283 rtx.payload_type = recv_codecs_[i].rtx_payload_type; 1284 } 1285 } 1286} 1287 1288bool WebRtcVideoChannel2::RemoveRecvStream(uint32 ssrc) { 1289 LOG(LS_INFO) << "RemoveRecvStream: " << ssrc; 1290 if (ssrc == 0) { 1291 LOG(LS_ERROR) << "RemoveRecvStream with 0 ssrc is not supported."; 1292 return false; 1293 } 1294 1295 rtc::CritScope stream_lock(&stream_crit_); 1296 std::map<uint32, WebRtcVideoReceiveStream*>::iterator stream = 1297 receive_streams_.find(ssrc); 1298 if (stream == receive_streams_.end()) { 1299 LOG(LS_ERROR) << "Stream not found for ssrc: " << ssrc; 1300 return false; 1301 } 1302 DeleteReceiveStream(stream->second); 1303 receive_streams_.erase(stream); 1304 1305 return true; 1306} 1307 1308bool WebRtcVideoChannel2::SetRenderer(uint32 ssrc, VideoRenderer* renderer) { 1309 LOG(LS_INFO) << "SetRenderer: ssrc:" << ssrc << " " 1310 << (renderer ? "(ptr)" : "NULL"); 1311 if (ssrc == 0) { 1312 default_unsignalled_ssrc_handler_.SetDefaultRenderer(this, renderer); 1313 return true; 1314 } 1315 1316 rtc::CritScope stream_lock(&stream_crit_); 1317 std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 1318 receive_streams_.find(ssrc); 1319 if (it == receive_streams_.end()) { 1320 return false; 1321 } 1322 1323 it->second->SetRenderer(renderer); 1324 return true; 1325} 1326 1327bool WebRtcVideoChannel2::GetRenderer(uint32 ssrc, VideoRenderer** renderer) { 1328 if (ssrc == 0) { 1329 *renderer = default_unsignalled_ssrc_handler_.GetDefaultRenderer(); 1330 return *renderer != NULL; 1331 } 1332 1333 rtc::CritScope stream_lock(&stream_crit_); 1334 std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 1335 receive_streams_.find(ssrc); 1336 if (it == receive_streams_.end()) { 1337 return false; 1338 } 1339 *renderer = it->second->GetRenderer(); 1340 return true; 1341} 1342 1343bool WebRtcVideoChannel2::GetStats(VideoMediaInfo* info) { 1344 info->Clear(); 1345 FillSenderStats(info); 1346 FillReceiverStats(info); 1347 webrtc::Call::Stats stats = call_->GetStats(); 1348 FillBandwidthEstimationStats(stats, info); 1349 if (stats.rtt_ms != -1) { 1350 for (size_t i = 0; i < info->senders.size(); ++i) { 1351 info->senders[i].rtt_ms = stats.rtt_ms; 1352 } 1353 } 1354 return true; 1355} 1356 1357void WebRtcVideoChannel2::FillSenderStats(VideoMediaInfo* video_media_info) { 1358 rtc::CritScope stream_lock(&stream_crit_); 1359 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1360 send_streams_.begin(); 1361 it != send_streams_.end(); 1362 ++it) { 1363 video_media_info->senders.push_back(it->second->GetVideoSenderInfo()); 1364 } 1365} 1366 1367void WebRtcVideoChannel2::FillReceiverStats(VideoMediaInfo* video_media_info) { 1368 rtc::CritScope stream_lock(&stream_crit_); 1369 for (std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 1370 receive_streams_.begin(); 1371 it != receive_streams_.end(); 1372 ++it) { 1373 video_media_info->receivers.push_back(it->second->GetVideoReceiverInfo()); 1374 } 1375} 1376 1377void WebRtcVideoChannel2::FillBandwidthEstimationStats( 1378 const webrtc::Call::Stats& stats, 1379 VideoMediaInfo* video_media_info) { 1380 BandwidthEstimationInfo bwe_info; 1381 bwe_info.available_send_bandwidth = stats.send_bandwidth_bps; 1382 bwe_info.available_recv_bandwidth = stats.recv_bandwidth_bps; 1383 bwe_info.bucket_delay = stats.pacer_delay_ms; 1384 1385 // Get send stream bitrate stats. 1386 rtc::CritScope stream_lock(&stream_crit_); 1387 for (std::map<uint32, WebRtcVideoSendStream*>::iterator stream = 1388 send_streams_.begin(); 1389 stream != send_streams_.end(); 1390 ++stream) { 1391 stream->second->FillBandwidthEstimationInfo(&bwe_info); 1392 } 1393 video_media_info->bw_estimations.push_back(bwe_info); 1394} 1395 1396bool WebRtcVideoChannel2::SetCapturer(uint32 ssrc, VideoCapturer* capturer) { 1397 LOG(LS_INFO) << "SetCapturer: " << ssrc << " -> " 1398 << (capturer != NULL ? "(capturer)" : "NULL"); 1399 DCHECK(ssrc != 0); 1400 { 1401 rtc::CritScope stream_lock(&stream_crit_); 1402 if (send_streams_.find(ssrc) == send_streams_.end()) { 1403 LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc; 1404 return false; 1405 } 1406 if (!send_streams_[ssrc]->SetCapturer(capturer)) { 1407 return false; 1408 } 1409 } 1410 1411 if (capturer) { 1412 capturer->SetApplyRotation( 1413 !FindHeaderExtension(send_rtp_extensions_, 1414 kRtpVideoRotationHeaderExtension)); 1415 } 1416 { 1417 rtc::CritScope lock(&capturer_crit_); 1418 capturers_[ssrc] = capturer; 1419 } 1420 return true; 1421} 1422 1423bool WebRtcVideoChannel2::SendIntraFrame() { 1424 // TODO(pbos): Implement. 1425 LOG(LS_VERBOSE) << "SendIntraFrame()."; 1426 return true; 1427} 1428 1429bool WebRtcVideoChannel2::RequestIntraFrame() { 1430 // TODO(pbos): Implement. 1431 LOG(LS_VERBOSE) << "SendIntraFrame()."; 1432 return true; 1433} 1434 1435void WebRtcVideoChannel2::OnPacketReceived( 1436 rtc::Buffer* packet, 1437 const rtc::PacketTime& packet_time) { 1438 const webrtc::PacketReceiver::DeliveryStatus delivery_result = 1439 call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, 1440 reinterpret_cast<const uint8_t*>(packet->data()), packet->size()); 1441 switch (delivery_result) { 1442 case webrtc::PacketReceiver::DELIVERY_OK: 1443 return; 1444 case webrtc::PacketReceiver::DELIVERY_PACKET_ERROR: 1445 return; 1446 case webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC: 1447 break; 1448 } 1449 1450 uint32 ssrc = 0; 1451 if (!GetRtpSsrc(packet->data(), packet->size(), &ssrc)) { 1452 return; 1453 } 1454 1455 int payload_type = 0; 1456 if (!GetRtpPayloadType(packet->data(), packet->size(), &payload_type)) { 1457 return; 1458 } 1459 1460 // See if this payload_type is registered as one that usually gets its own 1461 // SSRC (RTX) or at least is safe to drop either way (ULPFEC). If it is, and 1462 // it wasn't handled above by DeliverPacket, that means we don't know what 1463 // stream it associates with, and we shouldn't ever create an implicit channel 1464 // for these. 1465 for (auto& codec : recv_codecs_) { 1466 if (payload_type == codec.rtx_payload_type || 1467 payload_type == codec.fec.red_rtx_payload_type || 1468 payload_type == codec.fec.ulpfec_payload_type) { 1469 return; 1470 } 1471 } 1472 1473 switch (unsignalled_ssrc_handler_->OnUnsignalledSsrc(this, ssrc)) { 1474 case UnsignalledSsrcHandler::kDropPacket: 1475 return; 1476 case UnsignalledSsrcHandler::kDeliverPacket: 1477 break; 1478 } 1479 1480 if (call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, 1481 reinterpret_cast<const uint8_t*>(packet->data()), packet->size()) != 1482 webrtc::PacketReceiver::DELIVERY_OK) { 1483 LOG(LS_WARNING) << "Failed to deliver RTP packet on re-delivery."; 1484 return; 1485 } 1486} 1487 1488void WebRtcVideoChannel2::OnRtcpReceived( 1489 rtc::Buffer* packet, 1490 const rtc::PacketTime& packet_time) { 1491 if (call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, 1492 reinterpret_cast<const uint8_t*>(packet->data()), packet->size()) != 1493 webrtc::PacketReceiver::DELIVERY_OK) { 1494 LOG(LS_WARNING) << "Failed to deliver RTCP packet."; 1495 } 1496} 1497 1498void WebRtcVideoChannel2::OnReadyToSend(bool ready) { 1499 LOG(LS_VERBOSE) << "OnReadyToSend: " << (ready ? "Ready." : "Not ready."); 1500 call_->SignalNetworkState(ready ? webrtc::kNetworkUp : webrtc::kNetworkDown); 1501} 1502 1503bool WebRtcVideoChannel2::MuteStream(uint32 ssrc, bool mute) { 1504 LOG(LS_VERBOSE) << "MuteStream: " << ssrc << " -> " 1505 << (mute ? "mute" : "unmute"); 1506 DCHECK(ssrc != 0); 1507 rtc::CritScope stream_lock(&stream_crit_); 1508 if (send_streams_.find(ssrc) == send_streams_.end()) { 1509 LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc; 1510 return false; 1511 } 1512 1513 send_streams_[ssrc]->MuteStream(mute); 1514 return true; 1515} 1516 1517bool WebRtcVideoChannel2::SetRecvRtpHeaderExtensions( 1518 const std::vector<RtpHeaderExtension>& extensions) { 1519 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetRecvRtpHeaderExtensions"); 1520 LOG(LS_INFO) << "SetRecvRtpHeaderExtensions: " 1521 << RtpExtensionsToString(extensions); 1522 if (!ValidateRtpHeaderExtensionIds(extensions)) 1523 return false; 1524 1525 std::vector<webrtc::RtpExtension> filtered_extensions = 1526 FilterRtpExtensions(extensions); 1527 if (!RtpExtensionsHaveChanged(recv_rtp_extensions_, filtered_extensions)) { 1528 LOG(LS_INFO) << "Ignoring call to SetRecvRtpHeaderExtensions because " 1529 "header extensions haven't changed."; 1530 return true; 1531 } 1532 1533 recv_rtp_extensions_ = filtered_extensions; 1534 1535 rtc::CritScope stream_lock(&stream_crit_); 1536 for (std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 1537 receive_streams_.begin(); 1538 it != receive_streams_.end(); 1539 ++it) { 1540 it->second->SetRtpExtensions(recv_rtp_extensions_); 1541 } 1542 return true; 1543} 1544 1545bool WebRtcVideoChannel2::SetSendRtpHeaderExtensions( 1546 const std::vector<RtpHeaderExtension>& extensions) { 1547 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetSendRtpHeaderExtensions"); 1548 LOG(LS_INFO) << "SetSendRtpHeaderExtensions: " 1549 << RtpExtensionsToString(extensions); 1550 if (!ValidateRtpHeaderExtensionIds(extensions)) 1551 return false; 1552 1553 std::vector<webrtc::RtpExtension> filtered_extensions = 1554 FilterRtpExtensions(extensions); 1555 if (!RtpExtensionsHaveChanged(send_rtp_extensions_, filtered_extensions)) { 1556 LOG(LS_INFO) << "Ignoring call to SetSendRtpHeaderExtensions because " 1557 "header extensions haven't changed."; 1558 return true; 1559 } 1560 1561 send_rtp_extensions_ = filtered_extensions; 1562 1563 const webrtc::RtpExtension* cvo_extension = FindHeaderExtension( 1564 send_rtp_extensions_, kRtpVideoRotationHeaderExtension); 1565 1566 rtc::CritScope stream_lock(&stream_crit_); 1567 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1568 send_streams_.begin(); 1569 it != send_streams_.end(); 1570 ++it) { 1571 it->second->SetRtpExtensions(send_rtp_extensions_); 1572 it->second->SetApplyRotation(!cvo_extension); 1573 } 1574 return true; 1575} 1576 1577// Counter-intuitively this method doesn't only set global bitrate caps but also 1578// per-stream codec max bitrates. This is to permit SetMaxSendBitrate (b=AS) to 1579// raise bitrates above the 2000k default bitrate cap. 1580bool WebRtcVideoChannel2::SetMaxSendBandwidth(int max_bitrate_bps) { 1581 // TODO(pbos): Figure out whether b=AS means max bitrate for this 1582 // WebRtcVideoChannel2 (in which case we're good), or per sender (SSRC), in 1583 // which case this should not set a Call::BitrateConfig but rather reconfigure 1584 // all senders. 1585 LOG(LS_INFO) << "SetMaxSendBandwidth: " << max_bitrate_bps << "bps."; 1586 if (max_bitrate_bps == bitrate_config_.max_bitrate_bps) 1587 return true; 1588 1589 if (max_bitrate_bps <= 0) { 1590 // Unsetting max bitrate. 1591 max_bitrate_bps = -1; 1592 } 1593 bitrate_config_.start_bitrate_bps = -1; 1594 bitrate_config_.max_bitrate_bps = max_bitrate_bps; 1595 if (max_bitrate_bps > 0 && 1596 bitrate_config_.min_bitrate_bps > max_bitrate_bps) { 1597 bitrate_config_.min_bitrate_bps = max_bitrate_bps; 1598 } 1599 call_->SetBitrateConfig(bitrate_config_); 1600 rtc::CritScope stream_lock(&stream_crit_); 1601 for (auto& kv : send_streams_) 1602 kv.second->SetMaxBitrateBps(max_bitrate_bps); 1603 return true; 1604} 1605 1606bool WebRtcVideoChannel2::SetOptions(const VideoOptions& options) { 1607 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetOptions"); 1608 LOG(LS_INFO) << "SetOptions: " << options.ToString(); 1609 VideoOptions old_options = options_; 1610 options_.SetAll(options); 1611 if (options_ == old_options) { 1612 // No new options to set. 1613 return true; 1614 } 1615 { 1616 rtc::CritScope lock(&capturer_crit_); 1617 options_.cpu_overuse_detection.Get(&signal_cpu_adaptation_); 1618 } 1619 rtc::DiffServCodePoint dscp = options_.dscp.GetWithDefaultIfUnset(false) 1620 ? rtc::DSCP_AF41 1621 : rtc::DSCP_DEFAULT; 1622 MediaChannel::SetDscp(dscp); 1623 rtc::CritScope stream_lock(&stream_crit_); 1624 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1625 send_streams_.begin(); 1626 it != send_streams_.end(); 1627 ++it) { 1628 it->second->SetOptions(options_); 1629 } 1630 return true; 1631} 1632 1633void WebRtcVideoChannel2::SetInterface(NetworkInterface* iface) { 1634 MediaChannel::SetInterface(iface); 1635 // Set the RTP recv/send buffer to a bigger size 1636 MediaChannel::SetOption(NetworkInterface::ST_RTP, 1637 rtc::Socket::OPT_RCVBUF, 1638 kVideoRtpBufferSize); 1639 1640 // Speculative change to increase the outbound socket buffer size. 1641 // In b/15152257, we are seeing a significant number of packets discarded 1642 // due to lack of socket buffer space, although it's not yet clear what the 1643 // ideal value should be. 1644 MediaChannel::SetOption(NetworkInterface::ST_RTP, 1645 rtc::Socket::OPT_SNDBUF, 1646 kVideoRtpBufferSize); 1647} 1648 1649void WebRtcVideoChannel2::UpdateAspectRatio(int ratio_w, int ratio_h) { 1650 // TODO(pbos): Implement. 1651} 1652 1653void WebRtcVideoChannel2::OnMessage(rtc::Message* msg) { 1654 // Ignored. 1655} 1656 1657void WebRtcVideoChannel2::OnLoadUpdate(Load load) { 1658 // OnLoadUpdate can not take any locks that are held while creating streams 1659 // etc. Doing so establishes lock-order inversions between the webrtc process 1660 // thread on stream creation and locks such as stream_crit_ while calling out. 1661 rtc::CritScope stream_lock(&capturer_crit_); 1662 if (!signal_cpu_adaptation_) 1663 return; 1664 // Do not adapt resolution for screen content as this will likely result in 1665 // blurry and unreadable text. 1666 for (auto& kv : capturers_) { 1667 if (kv.second != nullptr 1668 && !kv.second->IsScreencast() 1669 && kv.second->video_adapter() != nullptr) { 1670 kv.second->video_adapter()->OnCpuResolutionRequest( 1671 load == kOveruse ? CoordinatedVideoAdapter::DOWNGRADE 1672 : CoordinatedVideoAdapter::UPGRADE); 1673 } 1674 } 1675} 1676 1677bool WebRtcVideoChannel2::SendRtp(const uint8_t* data, size_t len) { 1678 rtc::Buffer packet(data, len, kMaxRtpPacketLen); 1679 return MediaChannel::SendPacket(&packet); 1680} 1681 1682bool WebRtcVideoChannel2::SendRtcp(const uint8_t* data, size_t len) { 1683 rtc::Buffer packet(data, len, kMaxRtpPacketLen); 1684 return MediaChannel::SendRtcp(&packet); 1685} 1686 1687void WebRtcVideoChannel2::StartAllSendStreams() { 1688 rtc::CritScope stream_lock(&stream_crit_); 1689 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1690 send_streams_.begin(); 1691 it != send_streams_.end(); 1692 ++it) { 1693 it->second->Start(); 1694 } 1695} 1696 1697void WebRtcVideoChannel2::StopAllSendStreams() { 1698 rtc::CritScope stream_lock(&stream_crit_); 1699 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1700 send_streams_.begin(); 1701 it != send_streams_.end(); 1702 ++it) { 1703 it->second->Stop(); 1704 } 1705} 1706 1707WebRtcVideoChannel2::WebRtcVideoSendStream::VideoSendStreamParameters:: 1708 VideoSendStreamParameters( 1709 const webrtc::VideoSendStream::Config& config, 1710 const VideoOptions& options, 1711 int max_bitrate_bps, 1712 const Settable<VideoCodecSettings>& codec_settings) 1713 : config(config), 1714 options(options), 1715 max_bitrate_bps(max_bitrate_bps), 1716 codec_settings(codec_settings) { 1717} 1718 1719WebRtcVideoChannel2::WebRtcVideoSendStream::AllocatedEncoder::AllocatedEncoder( 1720 webrtc::VideoEncoder* encoder, 1721 webrtc::VideoCodecType type, 1722 bool external) 1723 : encoder(encoder), 1724 external_encoder(nullptr), 1725 type(type), 1726 external(external) { 1727 if (external) { 1728 external_encoder = encoder; 1729 this->encoder = 1730 new webrtc::VideoEncoderSoftwareFallbackWrapper(type, encoder); 1731 } 1732} 1733 1734WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream( 1735 webrtc::Call* call, 1736 const StreamParams& sp, 1737 const webrtc::VideoSendStream::Config& config, 1738 WebRtcVideoEncoderFactory* external_encoder_factory, 1739 const VideoOptions& options, 1740 int max_bitrate_bps, 1741 const Settable<VideoCodecSettings>& codec_settings, 1742 const std::vector<webrtc::RtpExtension>& rtp_extensions) 1743 : ssrcs_(sp.ssrcs), 1744 ssrc_groups_(sp.ssrc_groups), 1745 call_(call), 1746 external_encoder_factory_(external_encoder_factory), 1747 stream_(NULL), 1748 parameters_(config, options, max_bitrate_bps, codec_settings), 1749 allocated_encoder_(NULL, webrtc::kVideoCodecUnknown, false), 1750 capturer_(NULL), 1751 sending_(false), 1752 muted_(false), 1753 old_adapt_changes_(0), 1754 first_frame_timestamp_ms_(0), 1755 last_frame_timestamp_ms_(0) { 1756 parameters_.config.rtp.max_packet_size = kVideoMtu; 1757 1758 sp.GetPrimarySsrcs(¶meters_.config.rtp.ssrcs); 1759 sp.GetFidSsrcs(parameters_.config.rtp.ssrcs, 1760 ¶meters_.config.rtp.rtx.ssrcs); 1761 parameters_.config.rtp.c_name = sp.cname; 1762 parameters_.config.rtp.extensions = rtp_extensions; 1763 1764 VideoCodecSettings params; 1765 if (codec_settings.Get(¶ms)) { 1766 SetCodec(params); 1767 } 1768} 1769 1770WebRtcVideoChannel2::WebRtcVideoSendStream::~WebRtcVideoSendStream() { 1771 DisconnectCapturer(); 1772 if (stream_ != NULL) { 1773 call_->DestroyVideoSendStream(stream_); 1774 } 1775 DestroyVideoEncoder(&allocated_encoder_); 1776} 1777 1778static void CreateBlackFrame(webrtc::VideoFrame* video_frame, 1779 int width, 1780 int height) { 1781 video_frame->CreateEmptyFrame(width, height, width, (width + 1) / 2, 1782 (width + 1) / 2); 1783 memset(video_frame->buffer(webrtc::kYPlane), 16, 1784 video_frame->allocated_size(webrtc::kYPlane)); 1785 memset(video_frame->buffer(webrtc::kUPlane), 128, 1786 video_frame->allocated_size(webrtc::kUPlane)); 1787 memset(video_frame->buffer(webrtc::kVPlane), 128, 1788 video_frame->allocated_size(webrtc::kVPlane)); 1789} 1790 1791void WebRtcVideoChannel2::WebRtcVideoSendStream::InputFrame( 1792 VideoCapturer* capturer, 1793 const VideoFrame* frame) { 1794 TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::InputFrame"); 1795 webrtc::VideoFrame video_frame(frame->GetVideoFrameBuffer(), 0, 0, 1796 frame->GetVideoRotation()); 1797 rtc::CritScope cs(&lock_); 1798 if (stream_ == NULL) { 1799 // Frame input before send codecs are configured, dropping frame. 1800 return; 1801 } 1802 1803 // Not sending, abort early to prevent expensive reconfigurations while 1804 // setting up codecs etc. 1805 if (!sending_) 1806 return; 1807 1808 if (format_.width == 0) { // Dropping frames. 1809 DCHECK(format_.height == 0); 1810 LOG(LS_VERBOSE) << "VideoFormat 0x0 set, Dropping frame."; 1811 return; 1812 } 1813 if (muted_) { 1814 // Create a black frame to transmit instead. 1815 CreateBlackFrame(&video_frame, 1816 static_cast<int>(frame->GetWidth()), 1817 static_cast<int>(frame->GetHeight())); 1818 } 1819 1820 int64_t frame_delta_ms = frame->GetTimeStamp() / rtc::kNumNanosecsPerMillisec; 1821 // frame->GetTimeStamp() is essentially a delta, align to webrtc time 1822 if (first_frame_timestamp_ms_ == 0) { 1823 first_frame_timestamp_ms_ = rtc::Time() - frame_delta_ms; 1824 } 1825 1826 last_frame_timestamp_ms_ = first_frame_timestamp_ms_ + frame_delta_ms; 1827 video_frame.set_render_time_ms(last_frame_timestamp_ms_); 1828 // Reconfigure codec if necessary. 1829 SetDimensions( 1830 video_frame.width(), video_frame.height(), capturer->IsScreencast()); 1831 1832 stream_->Input()->IncomingCapturedFrame(video_frame); 1833} 1834 1835bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetCapturer( 1836 VideoCapturer* capturer) { 1837 TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetCapturer"); 1838 if (!DisconnectCapturer() && capturer == NULL) { 1839 return false; 1840 } 1841 1842 { 1843 rtc::CritScope cs(&lock_); 1844 1845 if (capturer == NULL) { 1846 if (stream_ != NULL) { 1847 LOG(LS_VERBOSE) << "Disabling capturer, sending black frame."; 1848 webrtc::VideoFrame black_frame; 1849 1850 CreateBlackFrame(&black_frame, last_dimensions_.width, 1851 last_dimensions_.height); 1852 1853 // Force this black frame not to be dropped due to timestamp order 1854 // check. As IncomingCapturedFrame will drop the frame if this frame's 1855 // timestamp is less than or equal to last frame's timestamp, it is 1856 // necessary to give this black frame a larger timestamp than the 1857 // previous one. 1858 last_frame_timestamp_ms_ += 1859 format_.interval / rtc::kNumNanosecsPerMillisec; 1860 black_frame.set_render_time_ms(last_frame_timestamp_ms_); 1861 stream_->Input()->IncomingCapturedFrame(black_frame); 1862 } 1863 1864 capturer_ = NULL; 1865 return true; 1866 } 1867 1868 capturer_ = capturer; 1869 } 1870 // Lock cannot be held while connecting the capturer to prevent lock-order 1871 // violations. 1872 capturer->SignalVideoFrame.connect(this, &WebRtcVideoSendStream::InputFrame); 1873 return true; 1874} 1875 1876bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetVideoFormat( 1877 const VideoFormat& format) { 1878 if ((format.width == 0 || format.height == 0) && 1879 format.width != format.height) { 1880 LOG(LS_ERROR) << "Can't set VideoFormat, width or height is zero (but not " 1881 "both, 0x0 drops frames)."; 1882 return false; 1883 } 1884 1885 rtc::CritScope cs(&lock_); 1886 if (format.width == 0 && format.height == 0) { 1887 LOG(LS_INFO) 1888 << "0x0 resolution selected. Captured frames will be dropped for ssrc: " 1889 << parameters_.config.rtp.ssrcs[0] << "."; 1890 } else { 1891 // TODO(pbos): Fix me, this only affects the last stream! 1892 parameters_.encoder_config.streams.back().max_framerate = 1893 VideoFormat::IntervalToFps(format.interval); 1894 SetDimensions(format.width, format.height, false); 1895 } 1896 1897 format_ = format; 1898 return true; 1899} 1900 1901void WebRtcVideoChannel2::WebRtcVideoSendStream::MuteStream(bool mute) { 1902 rtc::CritScope cs(&lock_); 1903 muted_ = mute; 1904} 1905 1906bool WebRtcVideoChannel2::WebRtcVideoSendStream::DisconnectCapturer() { 1907 cricket::VideoCapturer* capturer; 1908 { 1909 rtc::CritScope cs(&lock_); 1910 if (capturer_ == NULL) 1911 return false; 1912 1913 if (capturer_->video_adapter() != nullptr) 1914 old_adapt_changes_ += capturer_->video_adapter()->adaptation_changes(); 1915 1916 capturer = capturer_; 1917 capturer_ = NULL; 1918 } 1919 capturer->SignalVideoFrame.disconnect(this); 1920 return true; 1921} 1922 1923const std::vector<uint32>& 1924WebRtcVideoChannel2::WebRtcVideoSendStream::GetSsrcs() const { 1925 return ssrcs_; 1926} 1927 1928void WebRtcVideoChannel2::WebRtcVideoSendStream::SetApplyRotation( 1929 bool apply_rotation) { 1930 rtc::CritScope cs(&lock_); 1931 if (capturer_ == NULL) 1932 return; 1933 1934 capturer_->SetApplyRotation(apply_rotation); 1935} 1936 1937void WebRtcVideoChannel2::WebRtcVideoSendStream::SetOptions( 1938 const VideoOptions& options) { 1939 rtc::CritScope cs(&lock_); 1940 VideoCodecSettings codec_settings; 1941 if (parameters_.codec_settings.Get(&codec_settings)) { 1942 LOG(LS_INFO) << "SetCodecAndOptions because of SetOptions; options=" 1943 << options.ToString(); 1944 SetCodecAndOptions(codec_settings, options); 1945 } else { 1946 parameters_.options = options; 1947 } 1948} 1949 1950void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodec( 1951 const VideoCodecSettings& codec_settings) { 1952 rtc::CritScope cs(&lock_); 1953 LOG(LS_INFO) << "SetCodecAndOptions because of SetCodec."; 1954 SetCodecAndOptions(codec_settings, parameters_.options); 1955} 1956 1957webrtc::VideoCodecType CodecTypeFromName(const std::string& name) { 1958 if (CodecNamesEq(name, kVp8CodecName)) { 1959 return webrtc::kVideoCodecVP8; 1960 } else if (CodecNamesEq(name, kVp9CodecName)) { 1961 return webrtc::kVideoCodecVP9; 1962 } else if (CodecNamesEq(name, kH264CodecName)) { 1963 return webrtc::kVideoCodecH264; 1964 } 1965 return webrtc::kVideoCodecUnknown; 1966} 1967 1968WebRtcVideoChannel2::WebRtcVideoSendStream::AllocatedEncoder 1969WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoder( 1970 const VideoCodec& codec) { 1971 webrtc::VideoCodecType type = CodecTypeFromName(codec.name); 1972 1973 // Do not re-create encoders of the same type. 1974 if (type == allocated_encoder_.type && allocated_encoder_.encoder != NULL) { 1975 return allocated_encoder_; 1976 } 1977 1978 if (external_encoder_factory_ != NULL) { 1979 webrtc::VideoEncoder* encoder = 1980 external_encoder_factory_->CreateVideoEncoder(type); 1981 if (encoder != NULL) { 1982 return AllocatedEncoder(encoder, type, true); 1983 } 1984 } 1985 1986 if (type == webrtc::kVideoCodecVP8) { 1987 return AllocatedEncoder( 1988 webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kVp8), type, false); 1989 } else if (type == webrtc::kVideoCodecVP9) { 1990 return AllocatedEncoder( 1991 webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kVp9), type, false); 1992 } else if (type == webrtc::kVideoCodecH264) { 1993 return AllocatedEncoder( 1994 webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kH264), type, false); 1995 } 1996 1997 // This shouldn't happen, we should not be trying to create something we don't 1998 // support. 1999 DCHECK(false); 2000 return AllocatedEncoder(NULL, webrtc::kVideoCodecUnknown, false); 2001} 2002 2003void WebRtcVideoChannel2::WebRtcVideoSendStream::DestroyVideoEncoder( 2004 AllocatedEncoder* encoder) { 2005 if (encoder->external) { 2006 external_encoder_factory_->DestroyVideoEncoder(encoder->external_encoder); 2007 } 2008 delete encoder->encoder; 2009} 2010 2011void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodecAndOptions( 2012 const VideoCodecSettings& codec_settings, 2013 const VideoOptions& options) { 2014 parameters_.encoder_config = 2015 CreateVideoEncoderConfig(last_dimensions_, codec_settings.codec); 2016 if (parameters_.encoder_config.streams.empty()) 2017 return; 2018 2019 format_ = VideoFormat(codec_settings.codec.width, 2020 codec_settings.codec.height, 2021 VideoFormat::FpsToInterval(30), 2022 FOURCC_I420); 2023 2024 AllocatedEncoder new_encoder = CreateVideoEncoder(codec_settings.codec); 2025 parameters_.config.encoder_settings.encoder = new_encoder.encoder; 2026 parameters_.config.encoder_settings.payload_name = codec_settings.codec.name; 2027 parameters_.config.encoder_settings.payload_type = codec_settings.codec.id; 2028 parameters_.config.rtp.fec = codec_settings.fec; 2029 2030 // Set RTX payload type if RTX is enabled. 2031 if (!parameters_.config.rtp.rtx.ssrcs.empty()) { 2032 if (codec_settings.rtx_payload_type == -1) { 2033 LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX " 2034 "payload type. Ignoring."; 2035 parameters_.config.rtp.rtx.ssrcs.clear(); 2036 } else { 2037 parameters_.config.rtp.rtx.payload_type = codec_settings.rtx_payload_type; 2038 } 2039 } 2040 2041 parameters_.config.rtp.nack.rtp_history_ms = 2042 HasNack(codec_settings.codec) ? kNackHistoryMs : 0; 2043 2044 options.suspend_below_min_bitrate.Get( 2045 ¶meters_.config.suspend_below_min_bitrate); 2046 2047 parameters_.codec_settings.Set(codec_settings); 2048 parameters_.options = options; 2049 2050 LOG(LS_INFO) 2051 << "RecreateWebRtcStream (send) because of SetCodecAndOptions; options=" 2052 << options.ToString(); 2053 RecreateWebRtcStream(); 2054 if (allocated_encoder_.encoder != new_encoder.encoder) { 2055 DestroyVideoEncoder(&allocated_encoder_); 2056 allocated_encoder_ = new_encoder; 2057 } 2058} 2059 2060void WebRtcVideoChannel2::WebRtcVideoSendStream::SetRtpExtensions( 2061 const std::vector<webrtc::RtpExtension>& rtp_extensions) { 2062 rtc::CritScope cs(&lock_); 2063 parameters_.config.rtp.extensions = rtp_extensions; 2064 if (stream_ != nullptr) { 2065 LOG(LS_INFO) << "RecreateWebRtcStream (send) because of SetRtpExtensions"; 2066 RecreateWebRtcStream(); 2067 } 2068} 2069 2070webrtc::VideoEncoderConfig 2071WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoderConfig( 2072 const Dimensions& dimensions, 2073 const VideoCodec& codec) const { 2074 webrtc::VideoEncoderConfig encoder_config; 2075 if (dimensions.is_screencast) { 2076 int screencast_min_bitrate_kbps; 2077 parameters_.options.screencast_min_bitrate.Get( 2078 &screencast_min_bitrate_kbps); 2079 encoder_config.min_transmit_bitrate_bps = 2080 screencast_min_bitrate_kbps * 1000; 2081 encoder_config.content_type = 2082 webrtc::VideoEncoderConfig::ContentType::kScreen; 2083 } else { 2084 encoder_config.min_transmit_bitrate_bps = 0; 2085 encoder_config.content_type = 2086 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo; 2087 } 2088 2089 // Restrict dimensions according to codec max. 2090 int width = dimensions.width; 2091 int height = dimensions.height; 2092 if (!dimensions.is_screencast) { 2093 if (codec.width < width) 2094 width = codec.width; 2095 if (codec.height < height) 2096 height = codec.height; 2097 } 2098 2099 VideoCodec clamped_codec = codec; 2100 clamped_codec.width = width; 2101 clamped_codec.height = height; 2102 2103 // By default, the stream count for the codec configuration should match the 2104 // number of negotiated ssrcs. But if the codec is blacklisted for simulcast 2105 // or a screencast, only configure a single stream. 2106 size_t stream_count = parameters_.config.rtp.ssrcs.size(); 2107 if (IsCodecBlacklistedForSimulcast(codec.name) || dimensions.is_screencast) { 2108 stream_count = 1; 2109 } 2110 2111 encoder_config.streams = 2112 CreateVideoStreams(clamped_codec, parameters_.options, 2113 parameters_.max_bitrate_bps, stream_count); 2114 2115 // Conference mode screencast uses 2 temporal layers split at 100kbit. 2116 if (parameters_.options.conference_mode.GetWithDefaultIfUnset(false) && 2117 dimensions.is_screencast && encoder_config.streams.size() == 1) { 2118 ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault(); 2119 2120 // For screenshare in conference mode, tl0 and tl1 bitrates are piggybacked 2121 // on the VideoCodec struct as target and max bitrates, respectively. 2122 // See eg. webrtc::VP8EncoderImpl::SetRates(). 2123 encoder_config.streams[0].target_bitrate_bps = 2124 config.tl0_bitrate_kbps * 1000; 2125 encoder_config.streams[0].max_bitrate_bps = config.tl1_bitrate_kbps * 1000; 2126 encoder_config.streams[0].temporal_layer_thresholds_bps.clear(); 2127 encoder_config.streams[0].temporal_layer_thresholds_bps.push_back( 2128 config.tl0_bitrate_kbps * 1000); 2129 } 2130 return encoder_config; 2131} 2132 2133void WebRtcVideoChannel2::WebRtcVideoSendStream::SetDimensions( 2134 int width, 2135 int height, 2136 bool is_screencast) { 2137 if (last_dimensions_.width == width && last_dimensions_.height == height && 2138 last_dimensions_.is_screencast == is_screencast) { 2139 // Configured using the same parameters, do not reconfigure. 2140 return; 2141 } 2142 LOG(LS_INFO) << "SetDimensions: " << width << "x" << height 2143 << (is_screencast ? " (screencast)" : " (not screencast)"); 2144 2145 last_dimensions_.width = width; 2146 last_dimensions_.height = height; 2147 last_dimensions_.is_screencast = is_screencast; 2148 2149 DCHECK(!parameters_.encoder_config.streams.empty()); 2150 2151 VideoCodecSettings codec_settings; 2152 parameters_.codec_settings.Get(&codec_settings); 2153 2154 webrtc::VideoEncoderConfig encoder_config = 2155 CreateVideoEncoderConfig(last_dimensions_, codec_settings.codec); 2156 2157 encoder_config.encoder_specific_settings = ConfigureVideoEncoderSettings( 2158 codec_settings.codec, parameters_.options, is_screencast); 2159 2160 bool stream_reconfigured = stream_->ReconfigureVideoEncoder(encoder_config); 2161 2162 encoder_config.encoder_specific_settings = NULL; 2163 2164 if (!stream_reconfigured) { 2165 LOG(LS_WARNING) << "Failed to reconfigure video encoder for dimensions: " 2166 << width << "x" << height; 2167 return; 2168 } 2169 2170 parameters_.encoder_config = encoder_config; 2171} 2172 2173void WebRtcVideoChannel2::WebRtcVideoSendStream::Start() { 2174 rtc::CritScope cs(&lock_); 2175 DCHECK(stream_ != NULL); 2176 stream_->Start(); 2177 sending_ = true; 2178} 2179 2180void WebRtcVideoChannel2::WebRtcVideoSendStream::Stop() { 2181 rtc::CritScope cs(&lock_); 2182 if (stream_ != NULL) { 2183 stream_->Stop(); 2184 } 2185 sending_ = false; 2186} 2187 2188VideoSenderInfo 2189WebRtcVideoChannel2::WebRtcVideoSendStream::GetVideoSenderInfo() { 2190 VideoSenderInfo info; 2191 webrtc::VideoSendStream::Stats stats; 2192 { 2193 rtc::CritScope cs(&lock_); 2194 for (uint32_t ssrc : parameters_.config.rtp.ssrcs) 2195 info.add_ssrc(ssrc); 2196 2197 VideoCodecSettings codec_settings; 2198 if (parameters_.codec_settings.Get(&codec_settings)) 2199 info.codec_name = codec_settings.codec.name; 2200 for (size_t i = 0; i < parameters_.encoder_config.streams.size(); ++i) { 2201 if (i == parameters_.encoder_config.streams.size() - 1) { 2202 info.preferred_bitrate += 2203 parameters_.encoder_config.streams[i].max_bitrate_bps; 2204 } else { 2205 info.preferred_bitrate += 2206 parameters_.encoder_config.streams[i].target_bitrate_bps; 2207 } 2208 } 2209 2210 if (stream_ == NULL) 2211 return info; 2212 2213 stats = stream_->GetStats(); 2214 2215 info.adapt_changes = old_adapt_changes_; 2216 info.adapt_reason = CoordinatedVideoAdapter::ADAPTREASON_NONE; 2217 2218 if (capturer_ != NULL) { 2219 if (!capturer_->IsMuted()) { 2220 VideoFormat last_captured_frame_format; 2221 capturer_->GetStats(&info.adapt_frame_drops, &info.effects_frame_drops, 2222 &info.capturer_frame_time, 2223 &last_captured_frame_format); 2224 info.input_frame_width = last_captured_frame_format.width; 2225 info.input_frame_height = last_captured_frame_format.height; 2226 } 2227 if (capturer_->video_adapter() != nullptr) { 2228 info.adapt_changes += capturer_->video_adapter()->adaptation_changes(); 2229 info.adapt_reason = capturer_->video_adapter()->adapt_reason(); 2230 } 2231 } 2232 } 2233 info.ssrc_groups = ssrc_groups_; 2234 info.framerate_input = stats.input_frame_rate; 2235 info.framerate_sent = stats.encode_frame_rate; 2236 info.avg_encode_ms = stats.avg_encode_time_ms; 2237 info.encode_usage_percent = stats.encode_usage_percent; 2238 2239 info.nominal_bitrate = stats.media_bitrate_bps; 2240 2241 info.send_frame_width = 0; 2242 info.send_frame_height = 0; 2243 for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it = 2244 stats.substreams.begin(); 2245 it != stats.substreams.end(); ++it) { 2246 // TODO(pbos): Wire up additional stats, such as padding bytes. 2247 webrtc::VideoSendStream::StreamStats stream_stats = it->second; 2248 info.bytes_sent += stream_stats.rtp_stats.transmitted.payload_bytes + 2249 stream_stats.rtp_stats.transmitted.header_bytes + 2250 stream_stats.rtp_stats.transmitted.padding_bytes; 2251 info.packets_sent += stream_stats.rtp_stats.transmitted.packets; 2252 info.packets_lost += stream_stats.rtcp_stats.cumulative_lost; 2253 if (stream_stats.width > info.send_frame_width) 2254 info.send_frame_width = stream_stats.width; 2255 if (stream_stats.height > info.send_frame_height) 2256 info.send_frame_height = stream_stats.height; 2257 info.firs_rcvd += stream_stats.rtcp_packet_type_counts.fir_packets; 2258 info.nacks_rcvd += stream_stats.rtcp_packet_type_counts.nack_packets; 2259 info.plis_rcvd += stream_stats.rtcp_packet_type_counts.pli_packets; 2260 } 2261 2262 if (!stats.substreams.empty()) { 2263 // TODO(pbos): Report fraction lost per SSRC. 2264 webrtc::VideoSendStream::StreamStats first_stream_stats = 2265 stats.substreams.begin()->second; 2266 info.fraction_lost = 2267 static_cast<float>(first_stream_stats.rtcp_stats.fraction_lost) / 2268 (1 << 8); 2269 } 2270 2271 return info; 2272} 2273 2274void WebRtcVideoChannel2::WebRtcVideoSendStream::FillBandwidthEstimationInfo( 2275 BandwidthEstimationInfo* bwe_info) { 2276 rtc::CritScope cs(&lock_); 2277 if (stream_ == NULL) { 2278 return; 2279 } 2280 webrtc::VideoSendStream::Stats stats = stream_->GetStats(); 2281 for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it = 2282 stats.substreams.begin(); 2283 it != stats.substreams.end(); ++it) { 2284 bwe_info->transmit_bitrate += it->second.total_bitrate_bps; 2285 bwe_info->retransmit_bitrate += it->second.retransmit_bitrate_bps; 2286 } 2287 bwe_info->target_enc_bitrate += stats.target_media_bitrate_bps; 2288 bwe_info->actual_enc_bitrate += stats.media_bitrate_bps; 2289} 2290 2291void WebRtcVideoChannel2::WebRtcVideoSendStream::SetMaxBitrateBps( 2292 int max_bitrate_bps) { 2293 rtc::CritScope cs(&lock_); 2294 parameters_.max_bitrate_bps = max_bitrate_bps; 2295 2296 // No need to reconfigure if the stream hasn't been configured yet. 2297 if (parameters_.encoder_config.streams.empty()) 2298 return; 2299 2300 // Force a stream reconfigure to set the new max bitrate. 2301 int width = last_dimensions_.width; 2302 last_dimensions_.width = 0; 2303 SetDimensions(width, last_dimensions_.height, last_dimensions_.is_screencast); 2304} 2305 2306void WebRtcVideoChannel2::WebRtcVideoSendStream::RecreateWebRtcStream() { 2307 if (stream_ != NULL) { 2308 call_->DestroyVideoSendStream(stream_); 2309 } 2310 2311 VideoCodecSettings codec_settings; 2312 parameters_.codec_settings.Get(&codec_settings); 2313 parameters_.encoder_config.encoder_specific_settings = 2314 ConfigureVideoEncoderSettings( 2315 codec_settings.codec, parameters_.options, 2316 parameters_.encoder_config.content_type == 2317 webrtc::VideoEncoderConfig::ContentType::kScreen); 2318 2319 webrtc::VideoSendStream::Config config = parameters_.config; 2320 if (!config.rtp.rtx.ssrcs.empty() && config.rtp.rtx.payload_type == -1) { 2321 LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX " 2322 "payload type the set codec. Ignoring RTX."; 2323 config.rtp.rtx.ssrcs.clear(); 2324 } 2325 stream_ = call_->CreateVideoSendStream(config, parameters_.encoder_config); 2326 2327 parameters_.encoder_config.encoder_specific_settings = NULL; 2328 2329 if (sending_) { 2330 stream_->Start(); 2331 } 2332} 2333 2334WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( 2335 webrtc::Call* call, 2336 const StreamParams& sp, 2337 const webrtc::VideoReceiveStream::Config& config, 2338 WebRtcVideoDecoderFactory* external_decoder_factory, 2339 bool default_stream, 2340 const std::vector<VideoCodecSettings>& recv_codecs) 2341 : call_(call), 2342 ssrcs_(sp.ssrcs), 2343 ssrc_groups_(sp.ssrc_groups), 2344 stream_(NULL), 2345 default_stream_(default_stream), 2346 config_(config), 2347 external_decoder_factory_(external_decoder_factory), 2348 renderer_(NULL), 2349 last_width_(-1), 2350 last_height_(-1), 2351 first_frame_timestamp_(-1), 2352 estimated_remote_start_ntp_time_ms_(0) { 2353 config_.renderer = this; 2354 // SetRecvCodecs will also reset (start) the VideoReceiveStream. 2355 LOG(LS_INFO) << "SetRecvCodecs (recv) because we are creating the receive " 2356 "stream for the first time: " 2357 << CodecSettingsVectorToString(recv_codecs); 2358 SetRecvCodecs(recv_codecs); 2359} 2360 2361WebRtcVideoChannel2::WebRtcVideoReceiveStream::AllocatedDecoder:: 2362 AllocatedDecoder(webrtc::VideoDecoder* decoder, 2363 webrtc::VideoCodecType type, 2364 bool external) 2365 : decoder(decoder), 2366 external_decoder(nullptr), 2367 type(type), 2368 external(external) { 2369 if (external) { 2370 external_decoder = decoder; 2371 this->decoder = 2372 new webrtc::VideoDecoderSoftwareFallbackWrapper(type, external_decoder); 2373 } 2374} 2375 2376WebRtcVideoChannel2::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() { 2377 call_->DestroyVideoReceiveStream(stream_); 2378 ClearDecoders(&allocated_decoders_); 2379} 2380 2381const std::vector<uint32>& 2382WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetSsrcs() const { 2383 return ssrcs_; 2384} 2385 2386WebRtcVideoChannel2::WebRtcVideoReceiveStream::AllocatedDecoder 2387WebRtcVideoChannel2::WebRtcVideoReceiveStream::CreateOrReuseVideoDecoder( 2388 std::vector<AllocatedDecoder>* old_decoders, 2389 const VideoCodec& codec) { 2390 webrtc::VideoCodecType type = CodecTypeFromName(codec.name); 2391 2392 for (size_t i = 0; i < old_decoders->size(); ++i) { 2393 if ((*old_decoders)[i].type == type) { 2394 AllocatedDecoder decoder = (*old_decoders)[i]; 2395 (*old_decoders)[i] = old_decoders->back(); 2396 old_decoders->pop_back(); 2397 return decoder; 2398 } 2399 } 2400 2401 if (external_decoder_factory_ != NULL) { 2402 webrtc::VideoDecoder* decoder = 2403 external_decoder_factory_->CreateVideoDecoder(type); 2404 if (decoder != NULL) { 2405 return AllocatedDecoder(decoder, type, true); 2406 } 2407 } 2408 2409 if (type == webrtc::kVideoCodecVP8) { 2410 return AllocatedDecoder( 2411 webrtc::VideoDecoder::Create(webrtc::VideoDecoder::kVp8), type, false); 2412 } 2413 2414 if (type == webrtc::kVideoCodecVP9) { 2415 return AllocatedDecoder( 2416 webrtc::VideoDecoder::Create(webrtc::VideoDecoder::kVp9), type, false); 2417 } 2418 2419 if (type == webrtc::kVideoCodecH264) { 2420 return AllocatedDecoder( 2421 webrtc::VideoDecoder::Create(webrtc::VideoDecoder::kH264), type, false); 2422 } 2423 2424 // This shouldn't happen, we should not be trying to create something we don't 2425 // support. 2426 DCHECK(false); 2427 return AllocatedDecoder(NULL, webrtc::kVideoCodecUnknown, false); 2428} 2429 2430void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRecvCodecs( 2431 const std::vector<VideoCodecSettings>& recv_codecs) { 2432 std::vector<AllocatedDecoder> old_decoders = allocated_decoders_; 2433 allocated_decoders_.clear(); 2434 config_.decoders.clear(); 2435 for (size_t i = 0; i < recv_codecs.size(); ++i) { 2436 AllocatedDecoder allocated_decoder = 2437 CreateOrReuseVideoDecoder(&old_decoders, recv_codecs[i].codec); 2438 allocated_decoders_.push_back(allocated_decoder); 2439 2440 webrtc::VideoReceiveStream::Decoder decoder; 2441 decoder.decoder = allocated_decoder.decoder; 2442 decoder.payload_type = recv_codecs[i].codec.id; 2443 decoder.payload_name = recv_codecs[i].codec.name; 2444 config_.decoders.push_back(decoder); 2445 } 2446 2447 // TODO(pbos): Reconfigure RTX based on incoming recv_codecs. 2448 config_.rtp.fec = recv_codecs.front().fec; 2449 config_.rtp.nack.rtp_history_ms = 2450 HasNack(recv_codecs.begin()->codec) ? kNackHistoryMs : 0; 2451 2452 ClearDecoders(&old_decoders); 2453 LOG(LS_INFO) << "RecreateWebRtcStream (recv) because of SetRecvCodecs: " 2454 << CodecSettingsVectorToString(recv_codecs); 2455 RecreateWebRtcStream(); 2456} 2457 2458void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetLocalSsrc( 2459 uint32_t local_ssrc) { 2460 // TODO(pbos): Consider turning this sanity check into a DCHECK. You should 2461 // not be able to create a sender with the same SSRC as a receiver, but right 2462 // now this can't be done due to unittests depending on receiving what they 2463 // are sending from the same MediaChannel. 2464 if (local_ssrc == config_.rtp.remote_ssrc) { 2465 LOG(LS_INFO) << "Ignoring call to SetLocalSsrc because parameters are " 2466 "unchanged; local_ssrc=" << local_ssrc; 2467 return; 2468 } 2469 2470 config_.rtp.local_ssrc = local_ssrc; 2471 LOG(LS_INFO) 2472 << "RecreateWebRtcStream (recv) because of SetLocalSsrc; local_ssrc=" 2473 << local_ssrc; 2474 RecreateWebRtcStream(); 2475} 2476 2477void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetNackAndRemb( 2478 bool nack_enabled, bool remb_enabled) { 2479 int nack_history_ms = nack_enabled ? kNackHistoryMs : 0; 2480 if (config_.rtp.nack.rtp_history_ms == nack_history_ms && 2481 config_.rtp.remb == remb_enabled) { 2482 LOG(LS_INFO) << "Ignoring call to SetNackAndRemb because parameters are " 2483 "unchanged; nack=" << nack_enabled 2484 << ", remb=" << remb_enabled; 2485 return; 2486 } 2487 config_.rtp.remb = remb_enabled; 2488 config_.rtp.nack.rtp_history_ms = nack_history_ms; 2489 LOG(LS_INFO) << "RecreateWebRtcStream (recv) because of SetNackAndRemb; nack=" 2490 << nack_enabled << ", remb=" << remb_enabled; 2491 RecreateWebRtcStream(); 2492} 2493 2494void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRtpExtensions( 2495 const std::vector<webrtc::RtpExtension>& extensions) { 2496 config_.rtp.extensions = extensions; 2497 LOG(LS_INFO) << "RecreateWebRtcStream (recv) because of SetRtpExtensions"; 2498 RecreateWebRtcStream(); 2499} 2500 2501void WebRtcVideoChannel2::WebRtcVideoReceiveStream::RecreateWebRtcStream() { 2502 if (stream_ != NULL) { 2503 call_->DestroyVideoReceiveStream(stream_); 2504 } 2505 stream_ = call_->CreateVideoReceiveStream(config_); 2506 stream_->Start(); 2507} 2508 2509void WebRtcVideoChannel2::WebRtcVideoReceiveStream::ClearDecoders( 2510 std::vector<AllocatedDecoder>* allocated_decoders) { 2511 for (size_t i = 0; i < allocated_decoders->size(); ++i) { 2512 if ((*allocated_decoders)[i].external) { 2513 external_decoder_factory_->DestroyVideoDecoder( 2514 (*allocated_decoders)[i].external_decoder); 2515 } 2516 delete (*allocated_decoders)[i].decoder; 2517 } 2518 allocated_decoders->clear(); 2519} 2520 2521void WebRtcVideoChannel2::WebRtcVideoReceiveStream::RenderFrame( 2522 const webrtc::VideoFrame& frame, 2523 int time_to_render_ms) { 2524 rtc::CritScope crit(&renderer_lock_); 2525 2526 if (first_frame_timestamp_ < 0) 2527 first_frame_timestamp_ = frame.timestamp(); 2528 int64_t rtp_time_elapsed_since_first_frame = 2529 (timestamp_wraparound_handler_.Unwrap(frame.timestamp()) - 2530 first_frame_timestamp_); 2531 int64_t elapsed_time_ms = rtp_time_elapsed_since_first_frame / 2532 (cricket::kVideoCodecClockrate / 1000); 2533 if (frame.ntp_time_ms() > 0) 2534 estimated_remote_start_ntp_time_ms_ = frame.ntp_time_ms() - elapsed_time_ms; 2535 2536 if (renderer_ == NULL) { 2537 LOG(LS_WARNING) << "VideoReceiveStream not connected to a VideoRenderer."; 2538 return; 2539 } 2540 2541 if (frame.width() != last_width_ || frame.height() != last_height_) { 2542 SetSize(frame.width(), frame.height()); 2543 } 2544 2545 const WebRtcVideoFrame render_frame( 2546 frame.video_frame_buffer(), 2547 elapsed_time_ms * rtc::kNumNanosecsPerMillisec, 2548 frame.render_time_ms() * rtc::kNumNanosecsPerMillisec, frame.rotation()); 2549 renderer_->RenderFrame(&render_frame); 2550} 2551 2552bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::IsTextureSupported() const { 2553 return true; 2554} 2555 2556bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::IsDefaultStream() const { 2557 return default_stream_; 2558} 2559 2560void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRenderer( 2561 cricket::VideoRenderer* renderer) { 2562 rtc::CritScope crit(&renderer_lock_); 2563 renderer_ = renderer; 2564 if (renderer_ != NULL && last_width_ != -1) { 2565 SetSize(last_width_, last_height_); 2566 } 2567} 2568 2569VideoRenderer* WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetRenderer() { 2570 // TODO(pbos): Remove GetRenderer and all uses of it, it's thread-unsafe by 2571 // design. 2572 rtc::CritScope crit(&renderer_lock_); 2573 return renderer_; 2574} 2575 2576void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetSize(int width, 2577 int height) { 2578 rtc::CritScope crit(&renderer_lock_); 2579 if (!renderer_->SetSize(width, height, 0)) { 2580 LOG(LS_ERROR) << "Could not set renderer size."; 2581 } 2582 last_width_ = width; 2583 last_height_ = height; 2584} 2585 2586VideoReceiverInfo 2587WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetVideoReceiverInfo() { 2588 VideoReceiverInfo info; 2589 info.ssrc_groups = ssrc_groups_; 2590 info.add_ssrc(config_.rtp.remote_ssrc); 2591 webrtc::VideoReceiveStream::Stats stats = stream_->GetStats(); 2592 info.bytes_rcvd = stats.rtp_stats.transmitted.payload_bytes + 2593 stats.rtp_stats.transmitted.header_bytes + 2594 stats.rtp_stats.transmitted.padding_bytes; 2595 info.packets_rcvd = stats.rtp_stats.transmitted.packets; 2596 info.packets_lost = stats.rtcp_stats.cumulative_lost; 2597 info.fraction_lost = 2598 static_cast<float>(stats.rtcp_stats.fraction_lost) / (1 << 8); 2599 2600 info.framerate_rcvd = stats.network_frame_rate; 2601 info.framerate_decoded = stats.decode_frame_rate; 2602 info.framerate_output = stats.render_frame_rate; 2603 2604 { 2605 rtc::CritScope frame_cs(&renderer_lock_); 2606 info.frame_width = last_width_; 2607 info.frame_height = last_height_; 2608 info.capture_start_ntp_time_ms = estimated_remote_start_ntp_time_ms_; 2609 } 2610 2611 info.decode_ms = stats.decode_ms; 2612 info.max_decode_ms = stats.max_decode_ms; 2613 info.current_delay_ms = stats.current_delay_ms; 2614 info.target_delay_ms = stats.target_delay_ms; 2615 info.jitter_buffer_ms = stats.jitter_buffer_ms; 2616 info.min_playout_delay_ms = stats.min_playout_delay_ms; 2617 info.render_delay_ms = stats.render_delay_ms; 2618 2619 info.firs_sent = stats.rtcp_packet_type_counts.fir_packets; 2620 info.plis_sent = stats.rtcp_packet_type_counts.pli_packets; 2621 info.nacks_sent = stats.rtcp_packet_type_counts.nack_packets; 2622 2623 return info; 2624} 2625 2626WebRtcVideoChannel2::VideoCodecSettings::VideoCodecSettings() 2627 : rtx_payload_type(-1) {} 2628 2629bool WebRtcVideoChannel2::VideoCodecSettings::operator==( 2630 const WebRtcVideoChannel2::VideoCodecSettings& other) const { 2631 return codec == other.codec && 2632 fec.ulpfec_payload_type == other.fec.ulpfec_payload_type && 2633 fec.red_payload_type == other.fec.red_payload_type && 2634 fec.red_rtx_payload_type == other.fec.red_rtx_payload_type && 2635 rtx_payload_type == other.rtx_payload_type; 2636} 2637 2638bool WebRtcVideoChannel2::VideoCodecSettings::operator!=( 2639 const WebRtcVideoChannel2::VideoCodecSettings& other) const { 2640 return !(*this == other); 2641} 2642 2643std::vector<WebRtcVideoChannel2::VideoCodecSettings> 2644WebRtcVideoChannel2::MapCodecs(const std::vector<VideoCodec>& codecs) { 2645 DCHECK(!codecs.empty()); 2646 2647 std::vector<VideoCodecSettings> video_codecs; 2648 std::map<int, bool> payload_used; 2649 std::map<int, VideoCodec::CodecType> payload_codec_type; 2650 // |rtx_mapping| maps video payload type to rtx payload type. 2651 std::map<int, int> rtx_mapping; 2652 2653 webrtc::FecConfig fec_settings; 2654 2655 for (size_t i = 0; i < codecs.size(); ++i) { 2656 const VideoCodec& in_codec = codecs[i]; 2657 int payload_type = in_codec.id; 2658 2659 if (payload_used[payload_type]) { 2660 LOG(LS_ERROR) << "Payload type already registered: " 2661 << in_codec.ToString(); 2662 return std::vector<VideoCodecSettings>(); 2663 } 2664 payload_used[payload_type] = true; 2665 payload_codec_type[payload_type] = in_codec.GetCodecType(); 2666 2667 switch (in_codec.GetCodecType()) { 2668 case VideoCodec::CODEC_RED: { 2669 // RED payload type, should not have duplicates. 2670 DCHECK(fec_settings.red_payload_type == -1); 2671 fec_settings.red_payload_type = in_codec.id; 2672 continue; 2673 } 2674 2675 case VideoCodec::CODEC_ULPFEC: { 2676 // ULPFEC payload type, should not have duplicates. 2677 DCHECK(fec_settings.ulpfec_payload_type == -1); 2678 fec_settings.ulpfec_payload_type = in_codec.id; 2679 continue; 2680 } 2681 2682 case VideoCodec::CODEC_RTX: { 2683 int associated_payload_type; 2684 if (!in_codec.GetParam(kCodecParamAssociatedPayloadType, 2685 &associated_payload_type) || 2686 !IsValidRtpPayloadType(associated_payload_type)) { 2687 LOG(LS_ERROR) 2688 << "RTX codec with invalid or no associated payload type: " 2689 << in_codec.ToString(); 2690 return std::vector<VideoCodecSettings>(); 2691 } 2692 rtx_mapping[associated_payload_type] = in_codec.id; 2693 continue; 2694 } 2695 2696 case VideoCodec::CODEC_VIDEO: 2697 break; 2698 } 2699 2700 video_codecs.push_back(VideoCodecSettings()); 2701 video_codecs.back().codec = in_codec; 2702 } 2703 2704 // One of these codecs should have been a video codec. Only having FEC 2705 // parameters into this code is a logic error. 2706 DCHECK(!video_codecs.empty()); 2707 2708 for (std::map<int, int>::const_iterator it = rtx_mapping.begin(); 2709 it != rtx_mapping.end(); 2710 ++it) { 2711 if (!payload_used[it->first]) { 2712 LOG(LS_ERROR) << "RTX mapped to payload not in codec list."; 2713 return std::vector<VideoCodecSettings>(); 2714 } 2715 if (payload_codec_type[it->first] != VideoCodec::CODEC_VIDEO && 2716 payload_codec_type[it->first] != VideoCodec::CODEC_RED) { 2717 LOG(LS_ERROR) << "RTX not mapped to regular video codec or RED codec."; 2718 return std::vector<VideoCodecSettings>(); 2719 } 2720 2721 if (it->first == fec_settings.red_payload_type) { 2722 fec_settings.red_rtx_payload_type = it->second; 2723 } 2724 } 2725 2726 for (size_t i = 0; i < video_codecs.size(); ++i) { 2727 video_codecs[i].fec = fec_settings; 2728 if (rtx_mapping[video_codecs[i].codec.id] != 0 && 2729 rtx_mapping[video_codecs[i].codec.id] != 2730 fec_settings.red_payload_type) { 2731 video_codecs[i].rtx_payload_type = rtx_mapping[video_codecs[i].codec.id]; 2732 } 2733 } 2734 2735 return video_codecs; 2736} 2737 2738} // namespace cricket 2739 2740#endif // HAVE_WEBRTC_VIDEO 2741