webrtcvideoengine2.cc revision be16f79818d7c21b747189b3e86d8d98add3e6b1
1/* 2 * libjingle 3 * Copyright 2014 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#ifdef HAVE_WEBRTC_VIDEO 29#include "talk/media/webrtc/webrtcvideoengine2.h" 30 31#include <algorithm> 32#include <set> 33#include <string> 34 35#include "talk/media/base/videocapturer.h" 36#include "talk/media/base/videorenderer.h" 37#include "talk/media/webrtc/constants.h" 38#include "talk/media/webrtc/simulcast.h" 39#include "talk/media/webrtc/webrtcvideoencoderfactory.h" 40#include "talk/media/webrtc/webrtcvideoframe.h" 41#include "talk/media/webrtc/webrtcvoiceengine.h" 42#include "webrtc/base/buffer.h" 43#include "webrtc/base/logging.h" 44#include "webrtc/base/stringutils.h" 45#include "webrtc/base/timeutils.h" 46#include "webrtc/call.h" 47#include "webrtc/modules/video_coding/codecs/h264/include/h264.h" 48#include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h" 49#include "webrtc/system_wrappers/interface/field_trial.h" 50#include "webrtc/system_wrappers/interface/trace_event.h" 51#include "webrtc/video_decoder.h" 52#include "webrtc/video_encoder.h" 53 54namespace cricket { 55namespace { 56 57// Wrap cricket::WebRtcVideoEncoderFactory as a webrtc::VideoEncoderFactory. 58class EncoderFactoryAdapter : public webrtc::VideoEncoderFactory { 59 public: 60 // EncoderFactoryAdapter doesn't take ownership of |factory|, which is owned 61 // by e.g. PeerConnectionFactory. 62 explicit EncoderFactoryAdapter(cricket::WebRtcVideoEncoderFactory* factory) 63 : factory_(factory) {} 64 virtual ~EncoderFactoryAdapter() {} 65 66 // Implement webrtc::VideoEncoderFactory. 67 webrtc::VideoEncoder* Create() override { 68 return factory_->CreateVideoEncoder(webrtc::kVideoCodecVP8); 69 } 70 71 void Destroy(webrtc::VideoEncoder* encoder) override { 72 return factory_->DestroyVideoEncoder(encoder); 73 } 74 75 private: 76 cricket::WebRtcVideoEncoderFactory* const factory_; 77}; 78 79// An encoder factory that wraps Create requests for simulcastable codec types 80// with a webrtc::SimulcastEncoderAdapter. Non simulcastable codec type 81// requests are just passed through to the contained encoder factory. 82class WebRtcSimulcastEncoderFactory 83 : public cricket::WebRtcVideoEncoderFactory { 84 public: 85 // WebRtcSimulcastEncoderFactory doesn't take ownership of |factory|, which is 86 // owned by e.g. PeerConnectionFactory. 87 explicit WebRtcSimulcastEncoderFactory( 88 cricket::WebRtcVideoEncoderFactory* factory) 89 : factory_(factory) {} 90 91 static bool UseSimulcastEncoderFactory( 92 const std::vector<VideoCodec>& codecs) { 93 // If any codec is VP8, use the simulcast factory. If asked to create a 94 // non-VP8 codec, we'll just return a contained factory encoder directly. 95 for (const auto& codec : codecs) { 96 if (codec.type == webrtc::kVideoCodecVP8) { 97 return true; 98 } 99 } 100 return false; 101 } 102 103 webrtc::VideoEncoder* CreateVideoEncoder( 104 webrtc::VideoCodecType type) override { 105 RTC_DCHECK(factory_ != NULL); 106 // If it's a codec type we can simulcast, create a wrapped encoder. 107 if (type == webrtc::kVideoCodecVP8) { 108 return new webrtc::SimulcastEncoderAdapter( 109 new EncoderFactoryAdapter(factory_)); 110 } 111 webrtc::VideoEncoder* encoder = factory_->CreateVideoEncoder(type); 112 if (encoder) { 113 non_simulcast_encoders_.push_back(encoder); 114 } 115 return encoder; 116 } 117 118 const std::vector<VideoCodec>& codecs() const override { 119 return factory_->codecs(); 120 } 121 122 bool EncoderTypeHasInternalSource( 123 webrtc::VideoCodecType type) const override { 124 return factory_->EncoderTypeHasInternalSource(type); 125 } 126 127 void DestroyVideoEncoder(webrtc::VideoEncoder* encoder) override { 128 // Check first to see if the encoder wasn't wrapped in a 129 // SimulcastEncoderAdapter. In that case, ask the factory to destroy it. 130 if (std::remove(non_simulcast_encoders_.begin(), 131 non_simulcast_encoders_.end(), 132 encoder) != non_simulcast_encoders_.end()) { 133 factory_->DestroyVideoEncoder(encoder); 134 return; 135 } 136 137 // Otherwise, SimulcastEncoderAdapter can be deleted directly, and will call 138 // DestroyVideoEncoder on the factory for individual encoder instances. 139 delete encoder; 140 } 141 142 private: 143 cricket::WebRtcVideoEncoderFactory* factory_; 144 // A list of encoders that were created without being wrapped in a 145 // SimulcastEncoderAdapter. 146 std::vector<webrtc::VideoEncoder*> non_simulcast_encoders_; 147}; 148 149bool CodecIsInternallySupported(const std::string& codec_name) { 150 if (CodecNamesEq(codec_name, kVp8CodecName)) { 151 return true; 152 } 153 if (CodecNamesEq(codec_name, kVp9CodecName)) { 154 const std::string group_name = 155 webrtc::field_trial::FindFullName("WebRTC-SupportVP9"); 156 return group_name == "Enabled" || group_name == "EnabledByFlag"; 157 } 158 if (CodecNamesEq(codec_name, kH264CodecName)) { 159 return webrtc::H264Encoder::IsSupported() && 160 webrtc::H264Decoder::IsSupported(); 161 } 162 return false; 163} 164 165void AddDefaultFeedbackParams(VideoCodec* codec) { 166 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamCcm, kRtcpFbCcmParamFir)); 167 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kParamValueEmpty)); 168 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kRtcpFbNackParamPli)); 169 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamRemb, kParamValueEmpty)); 170} 171 172static VideoCodec MakeVideoCodecWithDefaultFeedbackParams(int payload_type, 173 const char* name) { 174 VideoCodec codec(payload_type, name, kDefaultVideoMaxWidth, 175 kDefaultVideoMaxHeight, kDefaultVideoMaxFramerate, 0); 176 AddDefaultFeedbackParams(&codec); 177 return codec; 178} 179 180static std::string CodecVectorToString(const std::vector<VideoCodec>& codecs) { 181 std::stringstream out; 182 out << '{'; 183 for (size_t i = 0; i < codecs.size(); ++i) { 184 out << codecs[i].ToString(); 185 if (i != codecs.size() - 1) { 186 out << ", "; 187 } 188 } 189 out << '}'; 190 return out.str(); 191} 192 193static bool ValidateCodecFormats(const std::vector<VideoCodec>& codecs) { 194 bool has_video = false; 195 for (size_t i = 0; i < codecs.size(); ++i) { 196 if (!codecs[i].ValidateCodecFormat()) { 197 return false; 198 } 199 if (codecs[i].GetCodecType() == VideoCodec::CODEC_VIDEO) { 200 has_video = true; 201 } 202 } 203 if (!has_video) { 204 LOG(LS_ERROR) << "Setting codecs without a video codec is invalid: " 205 << CodecVectorToString(codecs); 206 return false; 207 } 208 return true; 209} 210 211static bool ValidateStreamParams(const StreamParams& sp) { 212 if (sp.ssrcs.empty()) { 213 LOG(LS_ERROR) << "No SSRCs in stream parameters: " << sp.ToString(); 214 return false; 215 } 216 217 std::vector<uint32_t> primary_ssrcs; 218 sp.GetPrimarySsrcs(&primary_ssrcs); 219 std::vector<uint32_t> rtx_ssrcs; 220 sp.GetFidSsrcs(primary_ssrcs, &rtx_ssrcs); 221 for (uint32_t rtx_ssrc : rtx_ssrcs) { 222 bool rtx_ssrc_present = false; 223 for (uint32_t sp_ssrc : sp.ssrcs) { 224 if (sp_ssrc == rtx_ssrc) { 225 rtx_ssrc_present = true; 226 break; 227 } 228 } 229 if (!rtx_ssrc_present) { 230 LOG(LS_ERROR) << "RTX SSRC '" << rtx_ssrc 231 << "' missing from StreamParams ssrcs: " << sp.ToString(); 232 return false; 233 } 234 } 235 if (!rtx_ssrcs.empty() && primary_ssrcs.size() != rtx_ssrcs.size()) { 236 LOG(LS_ERROR) 237 << "RTX SSRCs exist, but don't cover all SSRCs (unsupported): " 238 << sp.ToString(); 239 return false; 240 } 241 242 return true; 243} 244 245static std::string RtpExtensionsToString( 246 const std::vector<RtpHeaderExtension>& extensions) { 247 std::stringstream out; 248 out << '{'; 249 for (size_t i = 0; i < extensions.size(); ++i) { 250 out << "{" << extensions[i].uri << ": " << extensions[i].id << "}"; 251 if (i != extensions.size() - 1) { 252 out << ", "; 253 } 254 } 255 out << '}'; 256 return out.str(); 257} 258 259inline const webrtc::RtpExtension* FindHeaderExtension( 260 const std::vector<webrtc::RtpExtension>& extensions, 261 const std::string& name) { 262 for (const auto& kv : extensions) { 263 if (kv.name == name) { 264 return &kv; 265 } 266 } 267 return NULL; 268} 269 270// Merges two fec configs and logs an error if a conflict arises 271// such that merging in different order would trigger a different output. 272static void MergeFecConfig(const webrtc::FecConfig& other, 273 webrtc::FecConfig* output) { 274 if (other.ulpfec_payload_type != -1) { 275 if (output->ulpfec_payload_type != -1 && 276 output->ulpfec_payload_type != other.ulpfec_payload_type) { 277 LOG(LS_WARNING) << "Conflict merging ulpfec_payload_type configs: " 278 << output->ulpfec_payload_type << " and " 279 << other.ulpfec_payload_type; 280 } 281 output->ulpfec_payload_type = other.ulpfec_payload_type; 282 } 283 if (other.red_payload_type != -1) { 284 if (output->red_payload_type != -1 && 285 output->red_payload_type != other.red_payload_type) { 286 LOG(LS_WARNING) << "Conflict merging red_payload_type configs: " 287 << output->red_payload_type << " and " 288 << other.red_payload_type; 289 } 290 output->red_payload_type = other.red_payload_type; 291 } 292 if (other.red_rtx_payload_type != -1) { 293 if (output->red_rtx_payload_type != -1 && 294 output->red_rtx_payload_type != other.red_rtx_payload_type) { 295 LOG(LS_WARNING) << "Conflict merging red_rtx_payload_type configs: " 296 << output->red_rtx_payload_type << " and " 297 << other.red_rtx_payload_type; 298 } 299 output->red_rtx_payload_type = other.red_rtx_payload_type; 300 } 301} 302 303// Returns true if the given codec is disallowed from doing simulcast. 304bool IsCodecBlacklistedForSimulcast(const std::string& codec_name) { 305 return CodecNamesEq(codec_name, kH264CodecName); 306} 307 308// The selected thresholds for QVGA and VGA corresponded to a QP around 10. 309// The change in QP declined above the selected bitrates. 310static int GetMaxDefaultVideoBitrateKbps(int width, int height) { 311 if (width * height <= 320 * 240) { 312 return 600; 313 } else if (width * height <= 640 * 480) { 314 return 1700; 315 } else if (width * height <= 960 * 540) { 316 return 2000; 317 } else { 318 return 2500; 319 } 320} 321} // namespace 322 323// Constants defined in talk/media/webrtc/constants.h 324// TODO(pbos): Move these to a separate constants.cc file. 325const int kMinVideoBitrate = 30; 326const int kStartVideoBitrate = 300; 327 328const int kVideoMtu = 1200; 329const int kVideoRtpBufferSize = 65536; 330 331// This constant is really an on/off, lower-level configurable NACK history 332// duration hasn't been implemented. 333static const int kNackHistoryMs = 1000; 334 335static const int kDefaultQpMax = 56; 336 337static const int kDefaultRtcpReceiverReportSsrc = 1; 338 339std::vector<VideoCodec> DefaultVideoCodecList() { 340 std::vector<VideoCodec> codecs; 341 if (CodecIsInternallySupported(kVp9CodecName)) { 342 codecs.push_back(MakeVideoCodecWithDefaultFeedbackParams(kDefaultVp9PlType, 343 kVp9CodecName)); 344 // TODO(andresp): Add rtx codec for vp9 and verify it works. 345 } 346 codecs.push_back(MakeVideoCodecWithDefaultFeedbackParams(kDefaultVp8PlType, 347 kVp8CodecName)); 348 if (CodecIsInternallySupported(kH264CodecName)) { 349 codecs.push_back(MakeVideoCodecWithDefaultFeedbackParams(kDefaultH264PlType, 350 kH264CodecName)); 351 } 352 codecs.push_back( 353 VideoCodec::CreateRtxCodec(kDefaultRtxVp8PlType, kDefaultVp8PlType)); 354 codecs.push_back(VideoCodec(kDefaultRedPlType, kRedCodecName)); 355 codecs.push_back(VideoCodec(kDefaultUlpfecType, kUlpfecCodecName)); 356 return codecs; 357} 358 359static bool FindFirstMatchingCodec(const std::vector<VideoCodec>& codecs, 360 const VideoCodec& requested_codec, 361 VideoCodec* matching_codec) { 362 for (size_t i = 0; i < codecs.size(); ++i) { 363 if (requested_codec.Matches(codecs[i])) { 364 *matching_codec = codecs[i]; 365 return true; 366 } 367 } 368 return false; 369} 370 371static bool ValidateRtpHeaderExtensionIds( 372 const std::vector<RtpHeaderExtension>& extensions) { 373 std::set<int> extensions_used; 374 for (size_t i = 0; i < extensions.size(); ++i) { 375 if (extensions[i].id <= 0 || extensions[i].id >= 15 || 376 !extensions_used.insert(extensions[i].id).second) { 377 LOG(LS_ERROR) << "RTP extensions are with incorrect or duplicate ids."; 378 return false; 379 } 380 } 381 return true; 382} 383 384static bool CompareRtpHeaderExtensionIds( 385 const webrtc::RtpExtension& extension1, 386 const webrtc::RtpExtension& extension2) { 387 // Sorting on ID is sufficient, more than one extension per ID is unsupported. 388 return extension1.id > extension2.id; 389} 390 391static std::vector<webrtc::RtpExtension> FilterRtpExtensions( 392 const std::vector<RtpHeaderExtension>& extensions) { 393 std::vector<webrtc::RtpExtension> webrtc_extensions; 394 for (size_t i = 0; i < extensions.size(); ++i) { 395 // Unsupported extensions will be ignored. 396 if (webrtc::RtpExtension::IsSupportedForVideo(extensions[i].uri)) { 397 webrtc_extensions.push_back(webrtc::RtpExtension( 398 extensions[i].uri, extensions[i].id)); 399 } else { 400 LOG(LS_WARNING) << "Unsupported RTP extension: " << extensions[i].uri; 401 } 402 } 403 404 // Sort filtered headers to make sure that they can later be compared 405 // regardless of in which order they were entered. 406 std::sort(webrtc_extensions.begin(), webrtc_extensions.end(), 407 CompareRtpHeaderExtensionIds); 408 return webrtc_extensions; 409} 410 411static bool RtpExtensionsHaveChanged( 412 const std::vector<webrtc::RtpExtension>& before, 413 const std::vector<webrtc::RtpExtension>& after) { 414 if (before.size() != after.size()) 415 return true; 416 for (size_t i = 0; i < before.size(); ++i) { 417 if (before[i].id != after[i].id) 418 return true; 419 if (before[i].name != after[i].name) 420 return true; 421 } 422 return false; 423} 424 425std::vector<webrtc::VideoStream> 426WebRtcVideoChannel2::WebRtcVideoSendStream::CreateSimulcastVideoStreams( 427 const VideoCodec& codec, 428 const VideoOptions& options, 429 int max_bitrate_bps, 430 size_t num_streams) { 431 int max_qp = kDefaultQpMax; 432 codec.GetParam(kCodecParamMaxQuantization, &max_qp); 433 434 return GetSimulcastConfig( 435 num_streams, codec.width, codec.height, max_bitrate_bps, max_qp, 436 codec.framerate != 0 ? codec.framerate : kDefaultVideoMaxFramerate); 437} 438 439std::vector<webrtc::VideoStream> 440WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoStreams( 441 const VideoCodec& codec, 442 const VideoOptions& options, 443 int max_bitrate_bps, 444 size_t num_streams) { 445 int codec_max_bitrate_kbps; 446 if (codec.GetParam(kCodecParamMaxBitrate, &codec_max_bitrate_kbps)) { 447 max_bitrate_bps = codec_max_bitrate_kbps * 1000; 448 } 449 if (num_streams != 1) { 450 return CreateSimulcastVideoStreams(codec, options, max_bitrate_bps, 451 num_streams); 452 } 453 454 // For unset max bitrates set default bitrate for non-simulcast. 455 if (max_bitrate_bps <= 0) { 456 max_bitrate_bps = 457 GetMaxDefaultVideoBitrateKbps(codec.width, codec.height) * 1000; 458 } 459 460 webrtc::VideoStream stream; 461 stream.width = codec.width; 462 stream.height = codec.height; 463 stream.max_framerate = 464 codec.framerate != 0 ? codec.framerate : kDefaultVideoMaxFramerate; 465 466 stream.min_bitrate_bps = kMinVideoBitrate * 1000; 467 stream.target_bitrate_bps = stream.max_bitrate_bps = max_bitrate_bps; 468 469 int max_qp = kDefaultQpMax; 470 codec.GetParam(kCodecParamMaxQuantization, &max_qp); 471 stream.max_qp = max_qp; 472 std::vector<webrtc::VideoStream> streams; 473 streams.push_back(stream); 474 return streams; 475} 476 477void* WebRtcVideoChannel2::WebRtcVideoSendStream::ConfigureVideoEncoderSettings( 478 const VideoCodec& codec, 479 const VideoOptions& options, 480 bool is_screencast) { 481 // No automatic resizing when using simulcast or screencast. 482 bool automatic_resize = 483 !is_screencast && parameters_.config.rtp.ssrcs.size() == 1; 484 bool frame_dropping = !is_screencast; 485 bool denoising; 486 if (is_screencast) { 487 denoising = false; 488 } else { 489 options.video_noise_reduction.Get(&denoising); 490 } 491 492 if (CodecNamesEq(codec.name, kVp8CodecName)) { 493 encoder_settings_.vp8 = webrtc::VideoEncoder::GetDefaultVp8Settings(); 494 encoder_settings_.vp8.automaticResizeOn = automatic_resize; 495 encoder_settings_.vp8.denoisingOn = denoising; 496 encoder_settings_.vp8.frameDroppingOn = frame_dropping; 497 return &encoder_settings_.vp8; 498 } 499 if (CodecNamesEq(codec.name, kVp9CodecName)) { 500 encoder_settings_.vp9 = webrtc::VideoEncoder::GetDefaultVp9Settings(); 501 encoder_settings_.vp9.denoisingOn = denoising; 502 encoder_settings_.vp9.frameDroppingOn = frame_dropping; 503 return &encoder_settings_.vp9; 504 } 505 return NULL; 506} 507 508DefaultUnsignalledSsrcHandler::DefaultUnsignalledSsrcHandler() 509 : default_recv_ssrc_(0), default_renderer_(NULL) {} 510 511UnsignalledSsrcHandler::Action DefaultUnsignalledSsrcHandler::OnUnsignalledSsrc( 512 WebRtcVideoChannel2* channel, 513 uint32_t ssrc) { 514 if (default_recv_ssrc_ != 0) { // Already one default stream. 515 LOG(LS_WARNING) << "Unknown SSRC, but default receive stream already set."; 516 return kDropPacket; 517 } 518 519 StreamParams sp; 520 sp.ssrcs.push_back(ssrc); 521 LOG(LS_INFO) << "Creating default receive stream for SSRC=" << ssrc << "."; 522 if (!channel->AddRecvStream(sp, true)) { 523 LOG(LS_WARNING) << "Could not create default receive stream."; 524 } 525 526 channel->SetRenderer(ssrc, default_renderer_); 527 default_recv_ssrc_ = ssrc; 528 return kDeliverPacket; 529} 530 531VideoRenderer* DefaultUnsignalledSsrcHandler::GetDefaultRenderer() const { 532 return default_renderer_; 533} 534 535void DefaultUnsignalledSsrcHandler::SetDefaultRenderer( 536 VideoMediaChannel* channel, 537 VideoRenderer* renderer) { 538 default_renderer_ = renderer; 539 if (default_recv_ssrc_ != 0) { 540 channel->SetRenderer(default_recv_ssrc_, default_renderer_); 541 } 542} 543 544WebRtcVideoEngine2::WebRtcVideoEngine2() 545 : initialized_(false), 546 external_decoder_factory_(NULL), 547 external_encoder_factory_(NULL) { 548 LOG(LS_INFO) << "WebRtcVideoEngine2::WebRtcVideoEngine2()"; 549 video_codecs_ = GetSupportedCodecs(); 550 rtp_header_extensions_.push_back( 551 RtpHeaderExtension(kRtpTimestampOffsetHeaderExtension, 552 kRtpTimestampOffsetHeaderExtensionDefaultId)); 553 rtp_header_extensions_.push_back( 554 RtpHeaderExtension(kRtpAbsoluteSenderTimeHeaderExtension, 555 kRtpAbsoluteSenderTimeHeaderExtensionDefaultId)); 556 rtp_header_extensions_.push_back( 557 RtpHeaderExtension(kRtpVideoRotationHeaderExtension, 558 kRtpVideoRotationHeaderExtensionDefaultId)); 559 if (webrtc::field_trial::FindFullName("WebRTC-SendSideBwe") == "Enabled") { 560 rtp_header_extensions_.push_back(RtpHeaderExtension( 561 kRtpTransportSequenceNumberHeaderExtension, 562 kRtpTransportSequenceNumberHeaderExtensionDefaultId)); 563 } 564} 565 566WebRtcVideoEngine2::~WebRtcVideoEngine2() { 567 LOG(LS_INFO) << "WebRtcVideoEngine2::~WebRtcVideoEngine2"; 568} 569 570void WebRtcVideoEngine2::Init() { 571 LOG(LS_INFO) << "WebRtcVideoEngine2::Init"; 572 initialized_ = true; 573} 574 575bool WebRtcVideoEngine2::SetDefaultEncoderConfig( 576 const VideoEncoderConfig& config) { 577 const VideoCodec& codec = config.max_codec; 578 bool supports_codec = false; 579 for (size_t i = 0; i < video_codecs_.size(); ++i) { 580 if (CodecNamesEq(video_codecs_[i].name, codec.name)) { 581 video_codecs_[i].width = codec.width; 582 video_codecs_[i].height = codec.height; 583 video_codecs_[i].framerate = codec.framerate; 584 supports_codec = true; 585 break; 586 } 587 } 588 589 if (!supports_codec) { 590 LOG(LS_ERROR) << "SetDefaultEncoderConfig, codec not supported: " 591 << codec.ToString(); 592 return false; 593 } 594 595 return true; 596} 597 598WebRtcVideoChannel2* WebRtcVideoEngine2::CreateChannel( 599 webrtc::Call* call, 600 const VideoOptions& options) { 601 RTC_DCHECK(initialized_); 602 LOG(LS_INFO) << "CreateChannel. Options: " << options.ToString(); 603 return new WebRtcVideoChannel2(call, options, video_codecs_, 604 external_encoder_factory_, external_decoder_factory_); 605} 606 607const std::vector<VideoCodec>& WebRtcVideoEngine2::codecs() const { 608 return video_codecs_; 609} 610 611const std::vector<RtpHeaderExtension>& 612WebRtcVideoEngine2::rtp_header_extensions() const { 613 return rtp_header_extensions_; 614} 615 616void WebRtcVideoEngine2::SetLogging(int min_sev, const char* filter) { 617 // TODO(pbos): Set up logging. 618 LOG(LS_VERBOSE) << "SetLogging: " << min_sev << '"' << filter << '"'; 619 // if min_sev == -1, we keep the current log level. 620 if (min_sev < 0) { 621 RTC_DCHECK(min_sev == -1); 622 return; 623 } 624} 625 626void WebRtcVideoEngine2::SetExternalDecoderFactory( 627 WebRtcVideoDecoderFactory* decoder_factory) { 628 RTC_DCHECK(!initialized_); 629 external_decoder_factory_ = decoder_factory; 630} 631 632void WebRtcVideoEngine2::SetExternalEncoderFactory( 633 WebRtcVideoEncoderFactory* encoder_factory) { 634 RTC_DCHECK(!initialized_); 635 if (external_encoder_factory_ == encoder_factory) 636 return; 637 638 // No matter what happens we shouldn't hold on to a stale 639 // WebRtcSimulcastEncoderFactory. 640 simulcast_encoder_factory_.reset(); 641 642 if (encoder_factory && 643 WebRtcSimulcastEncoderFactory::UseSimulcastEncoderFactory( 644 encoder_factory->codecs())) { 645 simulcast_encoder_factory_.reset( 646 new WebRtcSimulcastEncoderFactory(encoder_factory)); 647 encoder_factory = simulcast_encoder_factory_.get(); 648 } 649 external_encoder_factory_ = encoder_factory; 650 651 video_codecs_ = GetSupportedCodecs(); 652} 653 654bool WebRtcVideoEngine2::EnableTimedRender() { 655 // TODO(pbos): Figure out whether this can be removed. 656 return true; 657} 658 659// Checks to see whether we comprehend and could receive a particular codec 660bool WebRtcVideoEngine2::FindCodec(const VideoCodec& in) { 661 // TODO(pbos): Probe encoder factory to figure out that the codec is supported 662 // if supported by the encoder factory. Add a corresponding test that fails 663 // with this code (that doesn't ask the factory). 664 for (size_t j = 0; j < video_codecs_.size(); ++j) { 665 VideoCodec codec(video_codecs_[j].id, video_codecs_[j].name, 0, 0, 0, 0); 666 if (codec.Matches(in)) { 667 return true; 668 } 669 } 670 return false; 671} 672 673// Tells whether the |requested| codec can be transmitted or not. If it can be 674// transmitted |out| is set with the best settings supported. Aspect ratio will 675// be set as close to |current|'s as possible. If not set |requested|'s 676// dimensions will be used for aspect ratio matching. 677bool WebRtcVideoEngine2::CanSendCodec(const VideoCodec& requested, 678 const VideoCodec& current, 679 VideoCodec* out) { 680 RTC_DCHECK(out != NULL); 681 682 if (requested.width != requested.height && 683 (requested.height == 0 || requested.width == 0)) { 684 // 0xn and nx0 are invalid resolutions. 685 return false; 686 } 687 688 VideoCodec matching_codec; 689 if (!FindFirstMatchingCodec(video_codecs_, requested, &matching_codec)) { 690 // Codec not supported. 691 return false; 692 } 693 694 out->id = requested.id; 695 out->name = requested.name; 696 out->preference = requested.preference; 697 out->params = requested.params; 698 out->framerate = std::min(requested.framerate, matching_codec.framerate); 699 out->params = requested.params; 700 out->feedback_params = requested.feedback_params; 701 out->width = requested.width; 702 out->height = requested.height; 703 if (requested.width == 0 && requested.height == 0) { 704 return true; 705 } 706 707 while (out->width > matching_codec.width) { 708 out->width /= 2; 709 out->height /= 2; 710 } 711 712 return out->width > 0 && out->height > 0; 713} 714 715// Ignore spammy trace messages, mostly from the stats API when we haven't 716// gotten RTCP info yet from the remote side. 717bool WebRtcVideoEngine2::ShouldIgnoreTrace(const std::string& trace) { 718 static const char* const kTracesToIgnore[] = {NULL}; 719 for (const char* const* p = kTracesToIgnore; *p; ++p) { 720 if (trace.find(*p) == 0) { 721 return true; 722 } 723 } 724 return false; 725} 726 727std::vector<VideoCodec> WebRtcVideoEngine2::GetSupportedCodecs() const { 728 std::vector<VideoCodec> supported_codecs = DefaultVideoCodecList(); 729 730 if (external_encoder_factory_ == NULL) { 731 return supported_codecs; 732 } 733 734 const std::vector<WebRtcVideoEncoderFactory::VideoCodec>& codecs = 735 external_encoder_factory_->codecs(); 736 for (size_t i = 0; i < codecs.size(); ++i) { 737 // Don't add internally-supported codecs twice. 738 if (CodecIsInternallySupported(codecs[i].name)) { 739 continue; 740 } 741 742 // External video encoders are given payloads 120-127. This also means that 743 // we only support up to 8 external payload types. 744 const int kExternalVideoPayloadTypeBase = 120; 745 size_t payload_type = kExternalVideoPayloadTypeBase + i; 746 RTC_DCHECK(payload_type < 128); 747 VideoCodec codec(static_cast<int>(payload_type), 748 codecs[i].name, 749 codecs[i].max_width, 750 codecs[i].max_height, 751 codecs[i].max_fps, 752 0); 753 754 AddDefaultFeedbackParams(&codec); 755 supported_codecs.push_back(codec); 756 } 757 return supported_codecs; 758} 759 760WebRtcVideoChannel2::WebRtcVideoChannel2( 761 webrtc::Call* call, 762 const VideoOptions& options, 763 const std::vector<VideoCodec>& recv_codecs, 764 WebRtcVideoEncoderFactory* external_encoder_factory, 765 WebRtcVideoDecoderFactory* external_decoder_factory) 766 : call_(call), 767 unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_), 768 external_encoder_factory_(external_encoder_factory), 769 external_decoder_factory_(external_decoder_factory) { 770 RTC_DCHECK(thread_checker_.CalledOnValidThread()); 771 SetDefaultOptions(); 772 options_.SetAll(options); 773 options_.cpu_overuse_detection.Get(&signal_cpu_adaptation_); 774 rtcp_receiver_report_ssrc_ = kDefaultRtcpReceiverReportSsrc; 775 sending_ = false; 776 default_send_ssrc_ = 0; 777 SetRecvCodecs(recv_codecs); 778} 779 780void WebRtcVideoChannel2::SetDefaultOptions() { 781 options_.cpu_overuse_detection.Set(true); 782 options_.dscp.Set(false); 783 options_.suspend_below_min_bitrate.Set(false); 784 options_.video_noise_reduction.Set(true); 785 options_.screencast_min_bitrate.Set(0); 786} 787 788WebRtcVideoChannel2::~WebRtcVideoChannel2() { 789 for (auto& kv : send_streams_) 790 delete kv.second; 791 for (auto& kv : receive_streams_) 792 delete kv.second; 793} 794 795bool WebRtcVideoChannel2::CodecIsExternallySupported( 796 const std::string& name) const { 797 if (external_encoder_factory_ == NULL) { 798 return false; 799 } 800 801 const std::vector<WebRtcVideoEncoderFactory::VideoCodec> external_codecs = 802 external_encoder_factory_->codecs(); 803 for (size_t c = 0; c < external_codecs.size(); ++c) { 804 if (CodecNamesEq(name, external_codecs[c].name)) { 805 return true; 806 } 807 } 808 return false; 809} 810 811std::vector<WebRtcVideoChannel2::VideoCodecSettings> 812WebRtcVideoChannel2::FilterSupportedCodecs( 813 const std::vector<WebRtcVideoChannel2::VideoCodecSettings>& mapped_codecs) 814 const { 815 std::vector<VideoCodecSettings> supported_codecs; 816 for (size_t i = 0; i < mapped_codecs.size(); ++i) { 817 const VideoCodecSettings& codec = mapped_codecs[i]; 818 if (CodecIsInternallySupported(codec.codec.name) || 819 CodecIsExternallySupported(codec.codec.name)) { 820 supported_codecs.push_back(codec); 821 } 822 } 823 return supported_codecs; 824} 825 826bool WebRtcVideoChannel2::ReceiveCodecsHaveChanged( 827 std::vector<VideoCodecSettings> before, 828 std::vector<VideoCodecSettings> after) { 829 if (before.size() != after.size()) { 830 return true; 831 } 832 // The receive codec order doesn't matter, so we sort the codecs before 833 // comparing. This is necessary because currently the 834 // only way to change the send codec is to munge SDP, which causes 835 // the receive codec list to change order, which causes the streams 836 // to be recreates which causes a "blink" of black video. In order 837 // to support munging the SDP in this way without recreating receive 838 // streams, we ignore the order of the received codecs so that 839 // changing the order doesn't cause this "blink". 840 auto comparison = 841 [](const VideoCodecSettings& codec1, const VideoCodecSettings& codec2) { 842 return codec1.codec.id > codec2.codec.id; 843 }; 844 std::sort(before.begin(), before.end(), comparison); 845 std::sort(after.begin(), after.end(), comparison); 846 for (size_t i = 0; i < before.size(); ++i) { 847 // For the same reason that we sort the codecs, we also ignore the 848 // preference. We don't want a preference change on the receive 849 // side to cause recreation of the stream. 850 before[i].codec.preference = 0; 851 after[i].codec.preference = 0; 852 if (before[i] != after[i]) { 853 return true; 854 } 855 } 856 return false; 857} 858 859bool WebRtcVideoChannel2::SetSendParameters(const VideoSendParameters& params) { 860 // TODO(pbos): Refactor this to only recreate the send streams once 861 // instead of 4 times. 862 return (SetSendCodecs(params.codecs) && 863 SetSendRtpHeaderExtensions(params.extensions) && 864 SetMaxSendBandwidth(params.max_bandwidth_bps) && 865 SetOptions(params.options)); 866} 867 868bool WebRtcVideoChannel2::SetRecvParameters(const VideoRecvParameters& params) { 869 // TODO(pbos): Refactor this to only recreate the recv streams once 870 // instead of twice. 871 return (SetRecvCodecs(params.codecs) && 872 SetRecvRtpHeaderExtensions(params.extensions)); 873} 874 875std::string WebRtcVideoChannel2::CodecSettingsVectorToString( 876 const std::vector<VideoCodecSettings>& codecs) { 877 std::stringstream out; 878 out << '{'; 879 for (size_t i = 0; i < codecs.size(); ++i) { 880 out << codecs[i].codec.ToString(); 881 if (i != codecs.size() - 1) { 882 out << ", "; 883 } 884 } 885 out << '}'; 886 return out.str(); 887} 888 889bool WebRtcVideoChannel2::SetRecvCodecs(const std::vector<VideoCodec>& codecs) { 890 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetRecvCodecs"); 891 LOG(LS_INFO) << "SetRecvCodecs: " << CodecVectorToString(codecs); 892 if (!ValidateCodecFormats(codecs)) { 893 return false; 894 } 895 896 const std::vector<VideoCodecSettings> mapped_codecs = MapCodecs(codecs); 897 if (mapped_codecs.empty()) { 898 LOG(LS_ERROR) << "SetRecvCodecs called without any video codecs."; 899 return false; 900 } 901 902 std::vector<VideoCodecSettings> supported_codecs = 903 FilterSupportedCodecs(mapped_codecs); 904 905 if (mapped_codecs.size() != supported_codecs.size()) { 906 LOG(LS_ERROR) << "SetRecvCodecs called with unsupported video codecs."; 907 return false; 908 } 909 910 // Prevent reconfiguration when setting identical receive codecs. 911 if (!ReceiveCodecsHaveChanged(recv_codecs_, supported_codecs)) { 912 LOG(LS_INFO) 913 << "Ignoring call to SetRecvCodecs because codecs haven't changed."; 914 return true; 915 } 916 917 LOG(LS_INFO) << "Changing recv codecs from " 918 << CodecSettingsVectorToString(recv_codecs_) << " to " 919 << CodecSettingsVectorToString(supported_codecs); 920 recv_codecs_ = supported_codecs; 921 922 rtc::CritScope stream_lock(&stream_crit_); 923 for (std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it = 924 receive_streams_.begin(); 925 it != receive_streams_.end(); ++it) { 926 it->second->SetRecvCodecs(recv_codecs_); 927 } 928 929 return true; 930} 931 932bool WebRtcVideoChannel2::SetSendCodecs(const std::vector<VideoCodec>& codecs) { 933 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetSendCodecs"); 934 LOG(LS_INFO) << "SetSendCodecs: " << CodecVectorToString(codecs); 935 if (!ValidateCodecFormats(codecs)) { 936 return false; 937 } 938 939 const std::vector<VideoCodecSettings> supported_codecs = 940 FilterSupportedCodecs(MapCodecs(codecs)); 941 942 if (supported_codecs.empty()) { 943 LOG(LS_ERROR) << "No video codecs supported."; 944 return false; 945 } 946 947 LOG(LS_INFO) << "Using codec: " << supported_codecs.front().codec.ToString(); 948 949 VideoCodecSettings old_codec; 950 if (send_codec_.Get(&old_codec) && supported_codecs.front() == old_codec) { 951 LOG(LS_INFO) << "Ignore call to SetSendCodecs because first supported " 952 "codec hasn't changed."; 953 // Using same codec, avoid reconfiguring. 954 return true; 955 } 956 957 send_codec_.Set(supported_codecs.front()); 958 959 rtc::CritScope stream_lock(&stream_crit_); 960 LOG(LS_INFO) << "Change the send codec because SetSendCodecs has a different " 961 "first supported codec."; 962 for (auto& kv : send_streams_) { 963 RTC_DCHECK(kv.second != nullptr); 964 kv.second->SetCodec(supported_codecs.front()); 965 } 966 LOG(LS_INFO) << "SetNackAndRemb on all the receive streams because the send " 967 "codec has changed."; 968 for (auto& kv : receive_streams_) { 969 RTC_DCHECK(kv.second != nullptr); 970 kv.second->SetNackAndRemb(HasNack(supported_codecs.front().codec), 971 HasRemb(supported_codecs.front().codec)); 972 } 973 974 // TODO(holmer): Changing the codec parameters shouldn't necessarily mean that 975 // we change the min/max of bandwidth estimation. Reevaluate this. 976 VideoCodec codec = supported_codecs.front().codec; 977 int bitrate_kbps; 978 if (codec.GetParam(kCodecParamMinBitrate, &bitrate_kbps) && 979 bitrate_kbps > 0) { 980 bitrate_config_.min_bitrate_bps = bitrate_kbps * 1000; 981 } else { 982 bitrate_config_.min_bitrate_bps = 0; 983 } 984 if (codec.GetParam(kCodecParamStartBitrate, &bitrate_kbps) && 985 bitrate_kbps > 0) { 986 bitrate_config_.start_bitrate_bps = bitrate_kbps * 1000; 987 } else { 988 // Do not reconfigure start bitrate unless it's specified and positive. 989 bitrate_config_.start_bitrate_bps = -1; 990 } 991 if (codec.GetParam(kCodecParamMaxBitrate, &bitrate_kbps) && 992 bitrate_kbps > 0) { 993 bitrate_config_.max_bitrate_bps = bitrate_kbps * 1000; 994 } else { 995 bitrate_config_.max_bitrate_bps = -1; 996 } 997 call_->SetBitrateConfig(bitrate_config_); 998 999 return true; 1000} 1001 1002bool WebRtcVideoChannel2::GetSendCodec(VideoCodec* codec) { 1003 VideoCodecSettings codec_settings; 1004 if (!send_codec_.Get(&codec_settings)) { 1005 LOG(LS_VERBOSE) << "GetSendCodec: No send codec set."; 1006 return false; 1007 } 1008 *codec = codec_settings.codec; 1009 return true; 1010} 1011 1012bool WebRtcVideoChannel2::SetSendStreamFormat(uint32_t ssrc, 1013 const VideoFormat& format) { 1014 LOG(LS_VERBOSE) << "SetSendStreamFormat:" << ssrc << " -> " 1015 << format.ToString(); 1016 rtc::CritScope stream_lock(&stream_crit_); 1017 if (send_streams_.find(ssrc) == send_streams_.end()) { 1018 return false; 1019 } 1020 return send_streams_[ssrc]->SetVideoFormat(format); 1021} 1022 1023bool WebRtcVideoChannel2::SetSend(bool send) { 1024 LOG(LS_VERBOSE) << "SetSend: " << (send ? "true" : "false"); 1025 if (send && !send_codec_.IsSet()) { 1026 LOG(LS_ERROR) << "SetSend(true) called before setting codec."; 1027 return false; 1028 } 1029 if (send) { 1030 StartAllSendStreams(); 1031 } else { 1032 StopAllSendStreams(); 1033 } 1034 sending_ = send; 1035 return true; 1036} 1037 1038bool WebRtcVideoChannel2::SetVideoSend(uint32_t ssrc, bool enable, 1039 const VideoOptions* options) { 1040 // TODO(solenberg): The state change should be fully rolled back if any one of 1041 // these calls fail. 1042 if (!MuteStream(ssrc, !enable)) { 1043 return false; 1044 } 1045 if (enable && options) { 1046 return SetOptions(*options); 1047 } else { 1048 return true; 1049 } 1050} 1051 1052bool WebRtcVideoChannel2::ValidateSendSsrcAvailability( 1053 const StreamParams& sp) const { 1054 for (uint32_t ssrc: sp.ssrcs) { 1055 if (send_ssrcs_.find(ssrc) != send_ssrcs_.end()) { 1056 LOG(LS_ERROR) << "Send stream with SSRC '" << ssrc << "' already exists."; 1057 return false; 1058 } 1059 } 1060 return true; 1061} 1062 1063bool WebRtcVideoChannel2::ValidateReceiveSsrcAvailability( 1064 const StreamParams& sp) const { 1065 for (uint32_t ssrc: sp.ssrcs) { 1066 if (receive_ssrcs_.find(ssrc) != receive_ssrcs_.end()) { 1067 LOG(LS_ERROR) << "Receive stream with SSRC '" << ssrc 1068 << "' already exists."; 1069 return false; 1070 } 1071 } 1072 return true; 1073} 1074 1075bool WebRtcVideoChannel2::AddSendStream(const StreamParams& sp) { 1076 LOG(LS_INFO) << "AddSendStream: " << sp.ToString(); 1077 if (!ValidateStreamParams(sp)) 1078 return false; 1079 1080 rtc::CritScope stream_lock(&stream_crit_); 1081 1082 if (!ValidateSendSsrcAvailability(sp)) 1083 return false; 1084 1085 for (uint32_t used_ssrc : sp.ssrcs) 1086 send_ssrcs_.insert(used_ssrc); 1087 1088 webrtc::VideoSendStream::Config config(this); 1089 config.overuse_callback = this; 1090 1091 WebRtcVideoSendStream* stream = 1092 new WebRtcVideoSendStream(call_, 1093 sp, 1094 config, 1095 external_encoder_factory_, 1096 options_, 1097 bitrate_config_.max_bitrate_bps, 1098 send_codec_, 1099 send_rtp_extensions_); 1100 1101 uint32_t ssrc = sp.first_ssrc(); 1102 RTC_DCHECK(ssrc != 0); 1103 send_streams_[ssrc] = stream; 1104 1105 if (rtcp_receiver_report_ssrc_ == kDefaultRtcpReceiverReportSsrc) { 1106 rtcp_receiver_report_ssrc_ = ssrc; 1107 LOG(LS_INFO) << "SetLocalSsrc on all the receive streams because we added " 1108 "a send stream."; 1109 for (auto& kv : receive_streams_) 1110 kv.second->SetLocalSsrc(ssrc); 1111 } 1112 if (default_send_ssrc_ == 0) { 1113 default_send_ssrc_ = ssrc; 1114 } 1115 if (sending_) { 1116 stream->Start(); 1117 } 1118 1119 return true; 1120} 1121 1122bool WebRtcVideoChannel2::RemoveSendStream(uint32_t ssrc) { 1123 LOG(LS_INFO) << "RemoveSendStream: " << ssrc; 1124 1125 if (ssrc == 0) { 1126 if (default_send_ssrc_ == 0) { 1127 LOG(LS_ERROR) << "No default send stream active."; 1128 return false; 1129 } 1130 1131 LOG(LS_VERBOSE) << "Removing default stream: " << default_send_ssrc_; 1132 ssrc = default_send_ssrc_; 1133 } 1134 1135 WebRtcVideoSendStream* removed_stream; 1136 { 1137 rtc::CritScope stream_lock(&stream_crit_); 1138 std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = 1139 send_streams_.find(ssrc); 1140 if (it == send_streams_.end()) { 1141 return false; 1142 } 1143 1144 for (uint32_t old_ssrc : it->second->GetSsrcs()) 1145 send_ssrcs_.erase(old_ssrc); 1146 1147 removed_stream = it->second; 1148 send_streams_.erase(it); 1149 } 1150 1151 delete removed_stream; 1152 1153 if (ssrc == default_send_ssrc_) { 1154 default_send_ssrc_ = 0; 1155 } 1156 1157 return true; 1158} 1159 1160void WebRtcVideoChannel2::DeleteReceiveStream( 1161 WebRtcVideoChannel2::WebRtcVideoReceiveStream* stream) { 1162 for (uint32_t old_ssrc : stream->GetSsrcs()) 1163 receive_ssrcs_.erase(old_ssrc); 1164 delete stream; 1165} 1166 1167bool WebRtcVideoChannel2::AddRecvStream(const StreamParams& sp) { 1168 return AddRecvStream(sp, false); 1169} 1170 1171bool WebRtcVideoChannel2::AddRecvStream(const StreamParams& sp, 1172 bool default_stream) { 1173 RTC_DCHECK(thread_checker_.CalledOnValidThread()); 1174 1175 LOG(LS_INFO) << "AddRecvStream" << (default_stream ? " (default stream)" : "") 1176 << ": " << sp.ToString(); 1177 if (!ValidateStreamParams(sp)) 1178 return false; 1179 1180 uint32_t ssrc = sp.first_ssrc(); 1181 RTC_DCHECK(ssrc != 0); // TODO(pbos): Is this ever valid? 1182 1183 rtc::CritScope stream_lock(&stream_crit_); 1184 // Remove running stream if this was a default stream. 1185 auto prev_stream = receive_streams_.find(ssrc); 1186 if (prev_stream != receive_streams_.end()) { 1187 if (default_stream || !prev_stream->second->IsDefaultStream()) { 1188 LOG(LS_ERROR) << "Receive stream for SSRC '" << ssrc 1189 << "' already exists."; 1190 return false; 1191 } 1192 DeleteReceiveStream(prev_stream->second); 1193 receive_streams_.erase(prev_stream); 1194 } 1195 1196 if (!ValidateReceiveSsrcAvailability(sp)) 1197 return false; 1198 1199 for (uint32_t used_ssrc : sp.ssrcs) 1200 receive_ssrcs_.insert(used_ssrc); 1201 1202 webrtc::VideoReceiveStream::Config config(this); 1203 ConfigureReceiverRtp(&config, sp); 1204 1205 // Set up A/V sync group based on sync label. 1206 config.sync_group = sp.sync_label; 1207 1208 config.rtp.remb = false; 1209 VideoCodecSettings send_codec; 1210 if (send_codec_.Get(&send_codec)) { 1211 config.rtp.remb = HasRemb(send_codec.codec); 1212 } 1213 1214 receive_streams_[ssrc] = new WebRtcVideoReceiveStream( 1215 call_, sp, config, external_decoder_factory_, default_stream, 1216 recv_codecs_); 1217 1218 return true; 1219} 1220 1221void WebRtcVideoChannel2::ConfigureReceiverRtp( 1222 webrtc::VideoReceiveStream::Config* config, 1223 const StreamParams& sp) const { 1224 uint32_t ssrc = sp.first_ssrc(); 1225 1226 config->rtp.remote_ssrc = ssrc; 1227 config->rtp.local_ssrc = rtcp_receiver_report_ssrc_; 1228 1229 config->rtp.extensions = recv_rtp_extensions_; 1230 1231 // TODO(pbos): This protection is against setting the same local ssrc as 1232 // remote which is not permitted by the lower-level API. RTCP requires a 1233 // corresponding sender SSRC. Figure out what to do when we don't have 1234 // (receive-only) or know a good local SSRC. 1235 if (config->rtp.remote_ssrc == config->rtp.local_ssrc) { 1236 if (config->rtp.local_ssrc != kDefaultRtcpReceiverReportSsrc) { 1237 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc; 1238 } else { 1239 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc + 1; 1240 } 1241 } 1242 1243 for (size_t i = 0; i < recv_codecs_.size(); ++i) { 1244 MergeFecConfig(recv_codecs_[i].fec, &config->rtp.fec); 1245 } 1246 1247 for (size_t i = 0; i < recv_codecs_.size(); ++i) { 1248 uint32_t rtx_ssrc; 1249 if (recv_codecs_[i].rtx_payload_type != -1 && 1250 sp.GetFidSsrc(ssrc, &rtx_ssrc)) { 1251 webrtc::VideoReceiveStream::Config::Rtp::Rtx& rtx = 1252 config->rtp.rtx[recv_codecs_[i].codec.id]; 1253 rtx.ssrc = rtx_ssrc; 1254 rtx.payload_type = recv_codecs_[i].rtx_payload_type; 1255 } 1256 } 1257} 1258 1259bool WebRtcVideoChannel2::RemoveRecvStream(uint32_t ssrc) { 1260 LOG(LS_INFO) << "RemoveRecvStream: " << ssrc; 1261 if (ssrc == 0) { 1262 LOG(LS_ERROR) << "RemoveRecvStream with 0 ssrc is not supported."; 1263 return false; 1264 } 1265 1266 rtc::CritScope stream_lock(&stream_crit_); 1267 std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator stream = 1268 receive_streams_.find(ssrc); 1269 if (stream == receive_streams_.end()) { 1270 LOG(LS_ERROR) << "Stream not found for ssrc: " << ssrc; 1271 return false; 1272 } 1273 DeleteReceiveStream(stream->second); 1274 receive_streams_.erase(stream); 1275 1276 return true; 1277} 1278 1279bool WebRtcVideoChannel2::SetRenderer(uint32_t ssrc, VideoRenderer* renderer) { 1280 LOG(LS_INFO) << "SetRenderer: ssrc:" << ssrc << " " 1281 << (renderer ? "(ptr)" : "NULL"); 1282 if (ssrc == 0) { 1283 default_unsignalled_ssrc_handler_.SetDefaultRenderer(this, renderer); 1284 return true; 1285 } 1286 1287 rtc::CritScope stream_lock(&stream_crit_); 1288 std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it = 1289 receive_streams_.find(ssrc); 1290 if (it == receive_streams_.end()) { 1291 return false; 1292 } 1293 1294 it->second->SetRenderer(renderer); 1295 return true; 1296} 1297 1298bool WebRtcVideoChannel2::GetRenderer(uint32_t ssrc, VideoRenderer** renderer) { 1299 if (ssrc == 0) { 1300 *renderer = default_unsignalled_ssrc_handler_.GetDefaultRenderer(); 1301 return *renderer != NULL; 1302 } 1303 1304 rtc::CritScope stream_lock(&stream_crit_); 1305 std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it = 1306 receive_streams_.find(ssrc); 1307 if (it == receive_streams_.end()) { 1308 return false; 1309 } 1310 *renderer = it->second->GetRenderer(); 1311 return true; 1312} 1313 1314bool WebRtcVideoChannel2::GetStats(VideoMediaInfo* info) { 1315 info->Clear(); 1316 FillSenderStats(info); 1317 FillReceiverStats(info); 1318 webrtc::Call::Stats stats = call_->GetStats(); 1319 FillBandwidthEstimationStats(stats, info); 1320 if (stats.rtt_ms != -1) { 1321 for (size_t i = 0; i < info->senders.size(); ++i) { 1322 info->senders[i].rtt_ms = stats.rtt_ms; 1323 } 1324 } 1325 return true; 1326} 1327 1328void WebRtcVideoChannel2::FillSenderStats(VideoMediaInfo* video_media_info) { 1329 rtc::CritScope stream_lock(&stream_crit_); 1330 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = 1331 send_streams_.begin(); 1332 it != send_streams_.end(); ++it) { 1333 video_media_info->senders.push_back(it->second->GetVideoSenderInfo()); 1334 } 1335} 1336 1337void WebRtcVideoChannel2::FillReceiverStats(VideoMediaInfo* video_media_info) { 1338 rtc::CritScope stream_lock(&stream_crit_); 1339 for (std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it = 1340 receive_streams_.begin(); 1341 it != receive_streams_.end(); ++it) { 1342 video_media_info->receivers.push_back(it->second->GetVideoReceiverInfo()); 1343 } 1344} 1345 1346void WebRtcVideoChannel2::FillBandwidthEstimationStats( 1347 const webrtc::Call::Stats& stats, 1348 VideoMediaInfo* video_media_info) { 1349 BandwidthEstimationInfo bwe_info; 1350 bwe_info.available_send_bandwidth = stats.send_bandwidth_bps; 1351 bwe_info.available_recv_bandwidth = stats.recv_bandwidth_bps; 1352 bwe_info.bucket_delay = stats.pacer_delay_ms; 1353 1354 // Get send stream bitrate stats. 1355 rtc::CritScope stream_lock(&stream_crit_); 1356 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator stream = 1357 send_streams_.begin(); 1358 stream != send_streams_.end(); ++stream) { 1359 stream->second->FillBandwidthEstimationInfo(&bwe_info); 1360 } 1361 video_media_info->bw_estimations.push_back(bwe_info); 1362} 1363 1364bool WebRtcVideoChannel2::SetCapturer(uint32_t ssrc, VideoCapturer* capturer) { 1365 LOG(LS_INFO) << "SetCapturer: " << ssrc << " -> " 1366 << (capturer != NULL ? "(capturer)" : "NULL"); 1367 RTC_DCHECK(ssrc != 0); 1368 { 1369 rtc::CritScope stream_lock(&stream_crit_); 1370 if (send_streams_.find(ssrc) == send_streams_.end()) { 1371 LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc; 1372 return false; 1373 } 1374 if (!send_streams_[ssrc]->SetCapturer(capturer)) { 1375 return false; 1376 } 1377 } 1378 1379 if (capturer) { 1380 capturer->SetApplyRotation( 1381 !FindHeaderExtension(send_rtp_extensions_, 1382 kRtpVideoRotationHeaderExtension)); 1383 } 1384 { 1385 rtc::CritScope lock(&capturer_crit_); 1386 capturers_[ssrc] = capturer; 1387 } 1388 return true; 1389} 1390 1391bool WebRtcVideoChannel2::SendIntraFrame() { 1392 // TODO(pbos): Implement. 1393 LOG(LS_VERBOSE) << "SendIntraFrame()."; 1394 return true; 1395} 1396 1397bool WebRtcVideoChannel2::RequestIntraFrame() { 1398 // TODO(pbos): Implement. 1399 LOG(LS_VERBOSE) << "SendIntraFrame()."; 1400 return true; 1401} 1402 1403void WebRtcVideoChannel2::OnPacketReceived( 1404 rtc::Buffer* packet, 1405 const rtc::PacketTime& packet_time) { 1406 const webrtc::PacketTime webrtc_packet_time(packet_time.timestamp, 1407 packet_time.not_before); 1408 const webrtc::PacketReceiver::DeliveryStatus delivery_result = 1409 call_->Receiver()->DeliverPacket( 1410 webrtc::MediaType::VIDEO, 1411 reinterpret_cast<const uint8_t*>(packet->data()), packet->size(), 1412 webrtc_packet_time); 1413 switch (delivery_result) { 1414 case webrtc::PacketReceiver::DELIVERY_OK: 1415 return; 1416 case webrtc::PacketReceiver::DELIVERY_PACKET_ERROR: 1417 return; 1418 case webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC: 1419 break; 1420 } 1421 1422 uint32_t ssrc = 0; 1423 if (!GetRtpSsrc(packet->data(), packet->size(), &ssrc)) { 1424 return; 1425 } 1426 1427 int payload_type = 0; 1428 if (!GetRtpPayloadType(packet->data(), packet->size(), &payload_type)) { 1429 return; 1430 } 1431 1432 // See if this payload_type is registered as one that usually gets its own 1433 // SSRC (RTX) or at least is safe to drop either way (ULPFEC). If it is, and 1434 // it wasn't handled above by DeliverPacket, that means we don't know what 1435 // stream it associates with, and we shouldn't ever create an implicit channel 1436 // for these. 1437 for (auto& codec : recv_codecs_) { 1438 if (payload_type == codec.rtx_payload_type || 1439 payload_type == codec.fec.red_rtx_payload_type || 1440 payload_type == codec.fec.ulpfec_payload_type) { 1441 return; 1442 } 1443 } 1444 1445 switch (unsignalled_ssrc_handler_->OnUnsignalledSsrc(this, ssrc)) { 1446 case UnsignalledSsrcHandler::kDropPacket: 1447 return; 1448 case UnsignalledSsrcHandler::kDeliverPacket: 1449 break; 1450 } 1451 1452 if (call_->Receiver()->DeliverPacket( 1453 webrtc::MediaType::VIDEO, 1454 reinterpret_cast<const uint8_t*>(packet->data()), packet->size(), 1455 webrtc_packet_time) != webrtc::PacketReceiver::DELIVERY_OK) { 1456 LOG(LS_WARNING) << "Failed to deliver RTP packet on re-delivery."; 1457 return; 1458 } 1459} 1460 1461void WebRtcVideoChannel2::OnRtcpReceived( 1462 rtc::Buffer* packet, 1463 const rtc::PacketTime& packet_time) { 1464 const webrtc::PacketTime webrtc_packet_time(packet_time.timestamp, 1465 packet_time.not_before); 1466 if (call_->Receiver()->DeliverPacket( 1467 webrtc::MediaType::VIDEO, 1468 reinterpret_cast<const uint8_t*>(packet->data()), packet->size(), 1469 webrtc_packet_time) != webrtc::PacketReceiver::DELIVERY_OK) { 1470 LOG(LS_WARNING) << "Failed to deliver RTCP packet."; 1471 } 1472} 1473 1474void WebRtcVideoChannel2::OnReadyToSend(bool ready) { 1475 LOG(LS_VERBOSE) << "OnReadyToSend: " << (ready ? "Ready." : "Not ready."); 1476 call_->SignalNetworkState(ready ? webrtc::kNetworkUp : webrtc::kNetworkDown); 1477} 1478 1479bool WebRtcVideoChannel2::MuteStream(uint32_t ssrc, bool mute) { 1480 LOG(LS_VERBOSE) << "MuteStream: " << ssrc << " -> " 1481 << (mute ? "mute" : "unmute"); 1482 RTC_DCHECK(ssrc != 0); 1483 rtc::CritScope stream_lock(&stream_crit_); 1484 if (send_streams_.find(ssrc) == send_streams_.end()) { 1485 LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc; 1486 return false; 1487 } 1488 1489 send_streams_[ssrc]->MuteStream(mute); 1490 return true; 1491} 1492 1493bool WebRtcVideoChannel2::SetRecvRtpHeaderExtensions( 1494 const std::vector<RtpHeaderExtension>& extensions) { 1495 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetRecvRtpHeaderExtensions"); 1496 LOG(LS_INFO) << "SetRecvRtpHeaderExtensions: " 1497 << RtpExtensionsToString(extensions); 1498 if (!ValidateRtpHeaderExtensionIds(extensions)) 1499 return false; 1500 1501 std::vector<webrtc::RtpExtension> filtered_extensions = 1502 FilterRtpExtensions(extensions); 1503 if (!RtpExtensionsHaveChanged(recv_rtp_extensions_, filtered_extensions)) { 1504 LOG(LS_INFO) << "Ignoring call to SetRecvRtpHeaderExtensions because " 1505 "header extensions haven't changed."; 1506 return true; 1507 } 1508 1509 recv_rtp_extensions_ = filtered_extensions; 1510 1511 rtc::CritScope stream_lock(&stream_crit_); 1512 for (std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it = 1513 receive_streams_.begin(); 1514 it != receive_streams_.end(); ++it) { 1515 it->second->SetRtpExtensions(recv_rtp_extensions_); 1516 } 1517 return true; 1518} 1519 1520bool WebRtcVideoChannel2::SetSendRtpHeaderExtensions( 1521 const std::vector<RtpHeaderExtension>& extensions) { 1522 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetSendRtpHeaderExtensions"); 1523 LOG(LS_INFO) << "SetSendRtpHeaderExtensions: " 1524 << RtpExtensionsToString(extensions); 1525 if (!ValidateRtpHeaderExtensionIds(extensions)) 1526 return false; 1527 1528 std::vector<webrtc::RtpExtension> filtered_extensions = 1529 FilterRtpExtensions(extensions); 1530 if (!RtpExtensionsHaveChanged(send_rtp_extensions_, filtered_extensions)) { 1531 LOG(LS_INFO) << "Ignoring call to SetSendRtpHeaderExtensions because " 1532 "header extensions haven't changed."; 1533 return true; 1534 } 1535 1536 send_rtp_extensions_ = filtered_extensions; 1537 1538 const webrtc::RtpExtension* cvo_extension = FindHeaderExtension( 1539 send_rtp_extensions_, kRtpVideoRotationHeaderExtension); 1540 1541 rtc::CritScope stream_lock(&stream_crit_); 1542 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = 1543 send_streams_.begin(); 1544 it != send_streams_.end(); ++it) { 1545 it->second->SetRtpExtensions(send_rtp_extensions_); 1546 it->second->SetApplyRotation(!cvo_extension); 1547 } 1548 return true; 1549} 1550 1551// Counter-intuitively this method doesn't only set global bitrate caps but also 1552// per-stream codec max bitrates. This is to permit SetMaxSendBitrate (b=AS) to 1553// raise bitrates above the 2000k default bitrate cap. 1554bool WebRtcVideoChannel2::SetMaxSendBandwidth(int max_bitrate_bps) { 1555 // TODO(pbos): Figure out whether b=AS means max bitrate for this 1556 // WebRtcVideoChannel2 (in which case we're good), or per sender (SSRC), in 1557 // which case this should not set a Call::BitrateConfig but rather reconfigure 1558 // all senders. 1559 LOG(LS_INFO) << "SetMaxSendBandwidth: " << max_bitrate_bps << "bps."; 1560 if (max_bitrate_bps == bitrate_config_.max_bitrate_bps) 1561 return true; 1562 1563 if (max_bitrate_bps < 0) { 1564 // Option not set. 1565 return true; 1566 } 1567 if (max_bitrate_bps == 0) { 1568 // Unsetting max bitrate. 1569 max_bitrate_bps = -1; 1570 } 1571 bitrate_config_.start_bitrate_bps = -1; 1572 bitrate_config_.max_bitrate_bps = max_bitrate_bps; 1573 if (max_bitrate_bps > 0 && 1574 bitrate_config_.min_bitrate_bps > max_bitrate_bps) { 1575 bitrate_config_.min_bitrate_bps = max_bitrate_bps; 1576 } 1577 call_->SetBitrateConfig(bitrate_config_); 1578 rtc::CritScope stream_lock(&stream_crit_); 1579 for (auto& kv : send_streams_) 1580 kv.second->SetMaxBitrateBps(max_bitrate_bps); 1581 return true; 1582} 1583 1584bool WebRtcVideoChannel2::SetOptions(const VideoOptions& options) { 1585 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetOptions"); 1586 LOG(LS_INFO) << "SetOptions: " << options.ToString(); 1587 VideoOptions old_options = options_; 1588 options_.SetAll(options); 1589 if (options_ == old_options) { 1590 // No new options to set. 1591 return true; 1592 } 1593 { 1594 rtc::CritScope lock(&capturer_crit_); 1595 options_.cpu_overuse_detection.Get(&signal_cpu_adaptation_); 1596 } 1597 rtc::DiffServCodePoint dscp = options_.dscp.GetWithDefaultIfUnset(false) 1598 ? rtc::DSCP_AF41 1599 : rtc::DSCP_DEFAULT; 1600 MediaChannel::SetDscp(dscp); 1601 rtc::CritScope stream_lock(&stream_crit_); 1602 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = 1603 send_streams_.begin(); 1604 it != send_streams_.end(); ++it) { 1605 it->second->SetOptions(options_); 1606 } 1607 return true; 1608} 1609 1610void WebRtcVideoChannel2::SetInterface(NetworkInterface* iface) { 1611 MediaChannel::SetInterface(iface); 1612 // Set the RTP recv/send buffer to a bigger size 1613 MediaChannel::SetOption(NetworkInterface::ST_RTP, 1614 rtc::Socket::OPT_RCVBUF, 1615 kVideoRtpBufferSize); 1616 1617 // Speculative change to increase the outbound socket buffer size. 1618 // In b/15152257, we are seeing a significant number of packets discarded 1619 // due to lack of socket buffer space, although it's not yet clear what the 1620 // ideal value should be. 1621 MediaChannel::SetOption(NetworkInterface::ST_RTP, 1622 rtc::Socket::OPT_SNDBUF, 1623 kVideoRtpBufferSize); 1624} 1625 1626void WebRtcVideoChannel2::UpdateAspectRatio(int ratio_w, int ratio_h) { 1627 // TODO(pbos): Implement. 1628} 1629 1630void WebRtcVideoChannel2::OnMessage(rtc::Message* msg) { 1631 // Ignored. 1632} 1633 1634void WebRtcVideoChannel2::OnLoadUpdate(Load load) { 1635 // OnLoadUpdate can not take any locks that are held while creating streams 1636 // etc. Doing so establishes lock-order inversions between the webrtc process 1637 // thread on stream creation and locks such as stream_crit_ while calling out. 1638 rtc::CritScope stream_lock(&capturer_crit_); 1639 if (!signal_cpu_adaptation_) 1640 return; 1641 // Do not adapt resolution for screen content as this will likely result in 1642 // blurry and unreadable text. 1643 for (auto& kv : capturers_) { 1644 if (kv.second != nullptr 1645 && !kv.second->IsScreencast() 1646 && kv.second->video_adapter() != nullptr) { 1647 kv.second->video_adapter()->OnCpuResolutionRequest( 1648 load == kOveruse ? CoordinatedVideoAdapter::DOWNGRADE 1649 : CoordinatedVideoAdapter::UPGRADE); 1650 } 1651 } 1652} 1653 1654bool WebRtcVideoChannel2::SendRtp(const uint8_t* data, 1655 size_t len, 1656 const webrtc::PacketOptions& options) { 1657 rtc::Buffer packet(data, len, kMaxRtpPacketLen); 1658 rtc::PacketOptions rtc_options; 1659 rtc_options.packet_id = options.packet_id; 1660 return MediaChannel::SendPacket(&packet, rtc_options); 1661} 1662 1663bool WebRtcVideoChannel2::SendRtcp(const uint8_t* data, size_t len) { 1664 rtc::Buffer packet(data, len, kMaxRtpPacketLen); 1665 return MediaChannel::SendRtcp(&packet, rtc::PacketOptions()); 1666} 1667 1668void WebRtcVideoChannel2::StartAllSendStreams() { 1669 rtc::CritScope stream_lock(&stream_crit_); 1670 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = 1671 send_streams_.begin(); 1672 it != send_streams_.end(); ++it) { 1673 it->second->Start(); 1674 } 1675} 1676 1677void WebRtcVideoChannel2::StopAllSendStreams() { 1678 rtc::CritScope stream_lock(&stream_crit_); 1679 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = 1680 send_streams_.begin(); 1681 it != send_streams_.end(); ++it) { 1682 it->second->Stop(); 1683 } 1684} 1685 1686WebRtcVideoChannel2::WebRtcVideoSendStream::VideoSendStreamParameters:: 1687 VideoSendStreamParameters( 1688 const webrtc::VideoSendStream::Config& config, 1689 const VideoOptions& options, 1690 int max_bitrate_bps, 1691 const Settable<VideoCodecSettings>& codec_settings) 1692 : config(config), 1693 options(options), 1694 max_bitrate_bps(max_bitrate_bps), 1695 codec_settings(codec_settings) { 1696} 1697 1698WebRtcVideoChannel2::WebRtcVideoSendStream::AllocatedEncoder::AllocatedEncoder( 1699 webrtc::VideoEncoder* encoder, 1700 webrtc::VideoCodecType type, 1701 bool external) 1702 : encoder(encoder), 1703 external_encoder(nullptr), 1704 type(type), 1705 external(external) { 1706 if (external) { 1707 external_encoder = encoder; 1708 this->encoder = 1709 new webrtc::VideoEncoderSoftwareFallbackWrapper(type, encoder); 1710 } 1711} 1712 1713WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream( 1714 webrtc::Call* call, 1715 const StreamParams& sp, 1716 const webrtc::VideoSendStream::Config& config, 1717 WebRtcVideoEncoderFactory* external_encoder_factory, 1718 const VideoOptions& options, 1719 int max_bitrate_bps, 1720 const Settable<VideoCodecSettings>& codec_settings, 1721 const std::vector<webrtc::RtpExtension>& rtp_extensions) 1722 : ssrcs_(sp.ssrcs), 1723 ssrc_groups_(sp.ssrc_groups), 1724 call_(call), 1725 external_encoder_factory_(external_encoder_factory), 1726 stream_(NULL), 1727 parameters_(config, options, max_bitrate_bps, codec_settings), 1728 allocated_encoder_(NULL, webrtc::kVideoCodecUnknown, false), 1729 capturer_(NULL), 1730 sending_(false), 1731 muted_(false), 1732 old_adapt_changes_(0), 1733 first_frame_timestamp_ms_(0), 1734 last_frame_timestamp_ms_(0) { 1735 parameters_.config.rtp.max_packet_size = kVideoMtu; 1736 1737 sp.GetPrimarySsrcs(¶meters_.config.rtp.ssrcs); 1738 sp.GetFidSsrcs(parameters_.config.rtp.ssrcs, 1739 ¶meters_.config.rtp.rtx.ssrcs); 1740 parameters_.config.rtp.c_name = sp.cname; 1741 parameters_.config.rtp.extensions = rtp_extensions; 1742 1743 VideoCodecSettings params; 1744 if (codec_settings.Get(¶ms)) { 1745 SetCodec(params); 1746 } 1747} 1748 1749WebRtcVideoChannel2::WebRtcVideoSendStream::~WebRtcVideoSendStream() { 1750 DisconnectCapturer(); 1751 if (stream_ != NULL) { 1752 call_->DestroyVideoSendStream(stream_); 1753 } 1754 DestroyVideoEncoder(&allocated_encoder_); 1755} 1756 1757static void CreateBlackFrame(webrtc::VideoFrame* video_frame, 1758 int width, 1759 int height) { 1760 video_frame->CreateEmptyFrame(width, height, width, (width + 1) / 2, 1761 (width + 1) / 2); 1762 memset(video_frame->buffer(webrtc::kYPlane), 16, 1763 video_frame->allocated_size(webrtc::kYPlane)); 1764 memset(video_frame->buffer(webrtc::kUPlane), 128, 1765 video_frame->allocated_size(webrtc::kUPlane)); 1766 memset(video_frame->buffer(webrtc::kVPlane), 128, 1767 video_frame->allocated_size(webrtc::kVPlane)); 1768} 1769 1770void WebRtcVideoChannel2::WebRtcVideoSendStream::InputFrame( 1771 VideoCapturer* capturer, 1772 const VideoFrame* frame) { 1773 TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::InputFrame"); 1774 webrtc::VideoFrame video_frame(frame->GetVideoFrameBuffer(), 0, 0, 1775 frame->GetVideoRotation()); 1776 rtc::CritScope cs(&lock_); 1777 if (stream_ == NULL) { 1778 // Frame input before send codecs are configured, dropping frame. 1779 return; 1780 } 1781 1782 // Not sending, abort early to prevent expensive reconfigurations while 1783 // setting up codecs etc. 1784 if (!sending_) 1785 return; 1786 1787 if (format_.width == 0) { // Dropping frames. 1788 RTC_DCHECK(format_.height == 0); 1789 LOG(LS_VERBOSE) << "VideoFormat 0x0 set, Dropping frame."; 1790 return; 1791 } 1792 if (muted_) { 1793 // Create a black frame to transmit instead. 1794 CreateBlackFrame(&video_frame, 1795 static_cast<int>(frame->GetWidth()), 1796 static_cast<int>(frame->GetHeight())); 1797 } 1798 1799 int64_t frame_delta_ms = frame->GetTimeStamp() / rtc::kNumNanosecsPerMillisec; 1800 // frame->GetTimeStamp() is essentially a delta, align to webrtc time 1801 if (first_frame_timestamp_ms_ == 0) { 1802 first_frame_timestamp_ms_ = rtc::Time() - frame_delta_ms; 1803 } 1804 1805 last_frame_timestamp_ms_ = first_frame_timestamp_ms_ + frame_delta_ms; 1806 video_frame.set_render_time_ms(last_frame_timestamp_ms_); 1807 // Reconfigure codec if necessary. 1808 SetDimensions( 1809 video_frame.width(), video_frame.height(), capturer->IsScreencast()); 1810 1811 stream_->Input()->IncomingCapturedFrame(video_frame); 1812} 1813 1814bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetCapturer( 1815 VideoCapturer* capturer) { 1816 TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetCapturer"); 1817 if (!DisconnectCapturer() && capturer == NULL) { 1818 return false; 1819 } 1820 1821 { 1822 rtc::CritScope cs(&lock_); 1823 1824 // Reset timestamps to realign new incoming frames to a webrtc timestamp. A 1825 // new capturer may have a different timestamp delta than the previous one. 1826 first_frame_timestamp_ms_ = 0; 1827 1828 if (capturer == NULL) { 1829 if (stream_ != NULL) { 1830 LOG(LS_VERBOSE) << "Disabling capturer, sending black frame."; 1831 webrtc::VideoFrame black_frame; 1832 1833 CreateBlackFrame(&black_frame, last_dimensions_.width, 1834 last_dimensions_.height); 1835 1836 // Force this black frame not to be dropped due to timestamp order 1837 // check. As IncomingCapturedFrame will drop the frame if this frame's 1838 // timestamp is less than or equal to last frame's timestamp, it is 1839 // necessary to give this black frame a larger timestamp than the 1840 // previous one. 1841 last_frame_timestamp_ms_ += 1842 format_.interval / rtc::kNumNanosecsPerMillisec; 1843 black_frame.set_render_time_ms(last_frame_timestamp_ms_); 1844 stream_->Input()->IncomingCapturedFrame(black_frame); 1845 } 1846 1847 capturer_ = NULL; 1848 return true; 1849 } 1850 1851 capturer_ = capturer; 1852 } 1853 // Lock cannot be held while connecting the capturer to prevent lock-order 1854 // violations. 1855 capturer->SignalVideoFrame.connect(this, &WebRtcVideoSendStream::InputFrame); 1856 return true; 1857} 1858 1859bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetVideoFormat( 1860 const VideoFormat& format) { 1861 if ((format.width == 0 || format.height == 0) && 1862 format.width != format.height) { 1863 LOG(LS_ERROR) << "Can't set VideoFormat, width or height is zero (but not " 1864 "both, 0x0 drops frames)."; 1865 return false; 1866 } 1867 1868 rtc::CritScope cs(&lock_); 1869 if (format.width == 0 && format.height == 0) { 1870 LOG(LS_INFO) 1871 << "0x0 resolution selected. Captured frames will be dropped for ssrc: " 1872 << parameters_.config.rtp.ssrcs[0] << "."; 1873 } else { 1874 // TODO(pbos): Fix me, this only affects the last stream! 1875 parameters_.encoder_config.streams.back().max_framerate = 1876 VideoFormat::IntervalToFps(format.interval); 1877 SetDimensions(format.width, format.height, false); 1878 } 1879 1880 format_ = format; 1881 return true; 1882} 1883 1884void WebRtcVideoChannel2::WebRtcVideoSendStream::MuteStream(bool mute) { 1885 rtc::CritScope cs(&lock_); 1886 muted_ = mute; 1887} 1888 1889bool WebRtcVideoChannel2::WebRtcVideoSendStream::DisconnectCapturer() { 1890 cricket::VideoCapturer* capturer; 1891 { 1892 rtc::CritScope cs(&lock_); 1893 if (capturer_ == NULL) 1894 return false; 1895 1896 if (capturer_->video_adapter() != nullptr) 1897 old_adapt_changes_ += capturer_->video_adapter()->adaptation_changes(); 1898 1899 capturer = capturer_; 1900 capturer_ = NULL; 1901 } 1902 capturer->SignalVideoFrame.disconnect(this); 1903 return true; 1904} 1905 1906const std::vector<uint32_t>& 1907WebRtcVideoChannel2::WebRtcVideoSendStream::GetSsrcs() const { 1908 return ssrcs_; 1909} 1910 1911void WebRtcVideoChannel2::WebRtcVideoSendStream::SetApplyRotation( 1912 bool apply_rotation) { 1913 rtc::CritScope cs(&lock_); 1914 if (capturer_ == NULL) 1915 return; 1916 1917 capturer_->SetApplyRotation(apply_rotation); 1918} 1919 1920void WebRtcVideoChannel2::WebRtcVideoSendStream::SetOptions( 1921 const VideoOptions& options) { 1922 rtc::CritScope cs(&lock_); 1923 VideoCodecSettings codec_settings; 1924 if (parameters_.codec_settings.Get(&codec_settings)) { 1925 LOG(LS_INFO) << "SetCodecAndOptions because of SetOptions; options=" 1926 << options.ToString(); 1927 SetCodecAndOptions(codec_settings, options); 1928 } else { 1929 parameters_.options = options; 1930 } 1931} 1932 1933void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodec( 1934 const VideoCodecSettings& codec_settings) { 1935 rtc::CritScope cs(&lock_); 1936 LOG(LS_INFO) << "SetCodecAndOptions because of SetCodec."; 1937 SetCodecAndOptions(codec_settings, parameters_.options); 1938} 1939 1940webrtc::VideoCodecType CodecTypeFromName(const std::string& name) { 1941 if (CodecNamesEq(name, kVp8CodecName)) { 1942 return webrtc::kVideoCodecVP8; 1943 } else if (CodecNamesEq(name, kVp9CodecName)) { 1944 return webrtc::kVideoCodecVP9; 1945 } else if (CodecNamesEq(name, kH264CodecName)) { 1946 return webrtc::kVideoCodecH264; 1947 } 1948 return webrtc::kVideoCodecUnknown; 1949} 1950 1951WebRtcVideoChannel2::WebRtcVideoSendStream::AllocatedEncoder 1952WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoder( 1953 const VideoCodec& codec) { 1954 webrtc::VideoCodecType type = CodecTypeFromName(codec.name); 1955 1956 // Do not re-create encoders of the same type. 1957 if (type == allocated_encoder_.type && allocated_encoder_.encoder != NULL) { 1958 return allocated_encoder_; 1959 } 1960 1961 if (external_encoder_factory_ != NULL) { 1962 webrtc::VideoEncoder* encoder = 1963 external_encoder_factory_->CreateVideoEncoder(type); 1964 if (encoder != NULL) { 1965 return AllocatedEncoder(encoder, type, true); 1966 } 1967 } 1968 1969 if (type == webrtc::kVideoCodecVP8) { 1970 return AllocatedEncoder( 1971 webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kVp8), type, false); 1972 } else if (type == webrtc::kVideoCodecVP9) { 1973 return AllocatedEncoder( 1974 webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kVp9), type, false); 1975 } else if (type == webrtc::kVideoCodecH264) { 1976 return AllocatedEncoder( 1977 webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kH264), type, false); 1978 } 1979 1980 // This shouldn't happen, we should not be trying to create something we don't 1981 // support. 1982 RTC_DCHECK(false); 1983 return AllocatedEncoder(NULL, webrtc::kVideoCodecUnknown, false); 1984} 1985 1986void WebRtcVideoChannel2::WebRtcVideoSendStream::DestroyVideoEncoder( 1987 AllocatedEncoder* encoder) { 1988 if (encoder->external) { 1989 external_encoder_factory_->DestroyVideoEncoder(encoder->external_encoder); 1990 } 1991 delete encoder->encoder; 1992} 1993 1994void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodecAndOptions( 1995 const VideoCodecSettings& codec_settings, 1996 const VideoOptions& options) { 1997 parameters_.encoder_config = 1998 CreateVideoEncoderConfig(last_dimensions_, codec_settings.codec); 1999 if (parameters_.encoder_config.streams.empty()) 2000 return; 2001 2002 format_ = VideoFormat(codec_settings.codec.width, 2003 codec_settings.codec.height, 2004 VideoFormat::FpsToInterval(30), 2005 FOURCC_I420); 2006 2007 AllocatedEncoder new_encoder = CreateVideoEncoder(codec_settings.codec); 2008 parameters_.config.encoder_settings.encoder = new_encoder.encoder; 2009 parameters_.config.encoder_settings.payload_name = codec_settings.codec.name; 2010 parameters_.config.encoder_settings.payload_type = codec_settings.codec.id; 2011 if (new_encoder.external) { 2012 webrtc::VideoCodecType type = CodecTypeFromName(codec_settings.codec.name); 2013 parameters_.config.encoder_settings.internal_source = 2014 external_encoder_factory_->EncoderTypeHasInternalSource(type); 2015 } 2016 parameters_.config.rtp.fec = codec_settings.fec; 2017 2018 // Set RTX payload type if RTX is enabled. 2019 if (!parameters_.config.rtp.rtx.ssrcs.empty()) { 2020 if (codec_settings.rtx_payload_type == -1) { 2021 LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX " 2022 "payload type. Ignoring."; 2023 parameters_.config.rtp.rtx.ssrcs.clear(); 2024 } else { 2025 parameters_.config.rtp.rtx.payload_type = codec_settings.rtx_payload_type; 2026 } 2027 } 2028 2029 parameters_.config.rtp.nack.rtp_history_ms = 2030 HasNack(codec_settings.codec) ? kNackHistoryMs : 0; 2031 2032 options.suspend_below_min_bitrate.Get( 2033 ¶meters_.config.suspend_below_min_bitrate); 2034 2035 parameters_.codec_settings.Set(codec_settings); 2036 parameters_.options = options; 2037 2038 LOG(LS_INFO) 2039 << "RecreateWebRtcStream (send) because of SetCodecAndOptions; options=" 2040 << options.ToString(); 2041 RecreateWebRtcStream(); 2042 if (allocated_encoder_.encoder != new_encoder.encoder) { 2043 DestroyVideoEncoder(&allocated_encoder_); 2044 allocated_encoder_ = new_encoder; 2045 } 2046} 2047 2048void WebRtcVideoChannel2::WebRtcVideoSendStream::SetRtpExtensions( 2049 const std::vector<webrtc::RtpExtension>& rtp_extensions) { 2050 rtc::CritScope cs(&lock_); 2051 parameters_.config.rtp.extensions = rtp_extensions; 2052 if (stream_ != nullptr) { 2053 LOG(LS_INFO) << "RecreateWebRtcStream (send) because of SetRtpExtensions"; 2054 RecreateWebRtcStream(); 2055 } 2056} 2057 2058webrtc::VideoEncoderConfig 2059WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoderConfig( 2060 const Dimensions& dimensions, 2061 const VideoCodec& codec) const { 2062 webrtc::VideoEncoderConfig encoder_config; 2063 if (dimensions.is_screencast) { 2064 int screencast_min_bitrate_kbps; 2065 parameters_.options.screencast_min_bitrate.Get( 2066 &screencast_min_bitrate_kbps); 2067 encoder_config.min_transmit_bitrate_bps = 2068 screencast_min_bitrate_kbps * 1000; 2069 encoder_config.content_type = 2070 webrtc::VideoEncoderConfig::ContentType::kScreen; 2071 } else { 2072 encoder_config.min_transmit_bitrate_bps = 0; 2073 encoder_config.content_type = 2074 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo; 2075 } 2076 2077 // Restrict dimensions according to codec max. 2078 int width = dimensions.width; 2079 int height = dimensions.height; 2080 if (!dimensions.is_screencast) { 2081 if (codec.width < width) 2082 width = codec.width; 2083 if (codec.height < height) 2084 height = codec.height; 2085 } 2086 2087 VideoCodec clamped_codec = codec; 2088 clamped_codec.width = width; 2089 clamped_codec.height = height; 2090 2091 // By default, the stream count for the codec configuration should match the 2092 // number of negotiated ssrcs. But if the codec is blacklisted for simulcast 2093 // or a screencast, only configure a single stream. 2094 size_t stream_count = parameters_.config.rtp.ssrcs.size(); 2095 if (IsCodecBlacklistedForSimulcast(codec.name) || dimensions.is_screencast) { 2096 stream_count = 1; 2097 } 2098 2099 encoder_config.streams = 2100 CreateVideoStreams(clamped_codec, parameters_.options, 2101 parameters_.max_bitrate_bps, stream_count); 2102 2103 // Conference mode screencast uses 2 temporal layers split at 100kbit. 2104 if (parameters_.options.conference_mode.GetWithDefaultIfUnset(false) && 2105 dimensions.is_screencast && encoder_config.streams.size() == 1) { 2106 ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault(); 2107 2108 // For screenshare in conference mode, tl0 and tl1 bitrates are piggybacked 2109 // on the VideoCodec struct as target and max bitrates, respectively. 2110 // See eg. webrtc::VP8EncoderImpl::SetRates(). 2111 encoder_config.streams[0].target_bitrate_bps = 2112 config.tl0_bitrate_kbps * 1000; 2113 encoder_config.streams[0].max_bitrate_bps = config.tl1_bitrate_kbps * 1000; 2114 encoder_config.streams[0].temporal_layer_thresholds_bps.clear(); 2115 encoder_config.streams[0].temporal_layer_thresholds_bps.push_back( 2116 config.tl0_bitrate_kbps * 1000); 2117 } 2118 return encoder_config; 2119} 2120 2121void WebRtcVideoChannel2::WebRtcVideoSendStream::SetDimensions( 2122 int width, 2123 int height, 2124 bool is_screencast) { 2125 if (last_dimensions_.width == width && last_dimensions_.height == height && 2126 last_dimensions_.is_screencast == is_screencast) { 2127 // Configured using the same parameters, do not reconfigure. 2128 return; 2129 } 2130 LOG(LS_INFO) << "SetDimensions: " << width << "x" << height 2131 << (is_screencast ? " (screencast)" : " (not screencast)"); 2132 2133 last_dimensions_.width = width; 2134 last_dimensions_.height = height; 2135 last_dimensions_.is_screencast = is_screencast; 2136 2137 RTC_DCHECK(!parameters_.encoder_config.streams.empty()); 2138 2139 VideoCodecSettings codec_settings; 2140 parameters_.codec_settings.Get(&codec_settings); 2141 2142 webrtc::VideoEncoderConfig encoder_config = 2143 CreateVideoEncoderConfig(last_dimensions_, codec_settings.codec); 2144 2145 encoder_config.encoder_specific_settings = ConfigureVideoEncoderSettings( 2146 codec_settings.codec, parameters_.options, is_screencast); 2147 2148 bool stream_reconfigured = stream_->ReconfigureVideoEncoder(encoder_config); 2149 2150 encoder_config.encoder_specific_settings = NULL; 2151 2152 if (!stream_reconfigured) { 2153 LOG(LS_WARNING) << "Failed to reconfigure video encoder for dimensions: " 2154 << width << "x" << height; 2155 return; 2156 } 2157 2158 parameters_.encoder_config = encoder_config; 2159} 2160 2161void WebRtcVideoChannel2::WebRtcVideoSendStream::Start() { 2162 rtc::CritScope cs(&lock_); 2163 RTC_DCHECK(stream_ != NULL); 2164 stream_->Start(); 2165 sending_ = true; 2166} 2167 2168void WebRtcVideoChannel2::WebRtcVideoSendStream::Stop() { 2169 rtc::CritScope cs(&lock_); 2170 if (stream_ != NULL) { 2171 stream_->Stop(); 2172 } 2173 sending_ = false; 2174} 2175 2176VideoSenderInfo 2177WebRtcVideoChannel2::WebRtcVideoSendStream::GetVideoSenderInfo() { 2178 VideoSenderInfo info; 2179 webrtc::VideoSendStream::Stats stats; 2180 { 2181 rtc::CritScope cs(&lock_); 2182 for (uint32_t ssrc : parameters_.config.rtp.ssrcs) 2183 info.add_ssrc(ssrc); 2184 2185 VideoCodecSettings codec_settings; 2186 if (parameters_.codec_settings.Get(&codec_settings)) 2187 info.codec_name = codec_settings.codec.name; 2188 for (size_t i = 0; i < parameters_.encoder_config.streams.size(); ++i) { 2189 if (i == parameters_.encoder_config.streams.size() - 1) { 2190 info.preferred_bitrate += 2191 parameters_.encoder_config.streams[i].max_bitrate_bps; 2192 } else { 2193 info.preferred_bitrate += 2194 parameters_.encoder_config.streams[i].target_bitrate_bps; 2195 } 2196 } 2197 2198 if (stream_ == NULL) 2199 return info; 2200 2201 stats = stream_->GetStats(); 2202 2203 info.adapt_changes = old_adapt_changes_; 2204 info.adapt_reason = CoordinatedVideoAdapter::ADAPTREASON_NONE; 2205 2206 if (capturer_ != NULL) { 2207 if (!capturer_->IsMuted()) { 2208 VideoFormat last_captured_frame_format; 2209 capturer_->GetStats(&info.adapt_frame_drops, &info.effects_frame_drops, 2210 &info.capturer_frame_time, 2211 &last_captured_frame_format); 2212 info.input_frame_width = last_captured_frame_format.width; 2213 info.input_frame_height = last_captured_frame_format.height; 2214 } 2215 if (capturer_->video_adapter() != nullptr) { 2216 info.adapt_changes += capturer_->video_adapter()->adaptation_changes(); 2217 info.adapt_reason = capturer_->video_adapter()->adapt_reason(); 2218 } 2219 } 2220 } 2221 info.ssrc_groups = ssrc_groups_; 2222 info.framerate_input = stats.input_frame_rate; 2223 info.framerate_sent = stats.encode_frame_rate; 2224 info.avg_encode_ms = stats.avg_encode_time_ms; 2225 info.encode_usage_percent = stats.encode_usage_percent; 2226 2227 info.nominal_bitrate = stats.media_bitrate_bps; 2228 2229 info.send_frame_width = 0; 2230 info.send_frame_height = 0; 2231 for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it = 2232 stats.substreams.begin(); 2233 it != stats.substreams.end(); ++it) { 2234 // TODO(pbos): Wire up additional stats, such as padding bytes. 2235 webrtc::VideoSendStream::StreamStats stream_stats = it->second; 2236 info.bytes_sent += stream_stats.rtp_stats.transmitted.payload_bytes + 2237 stream_stats.rtp_stats.transmitted.header_bytes + 2238 stream_stats.rtp_stats.transmitted.padding_bytes; 2239 info.packets_sent += stream_stats.rtp_stats.transmitted.packets; 2240 info.packets_lost += stream_stats.rtcp_stats.cumulative_lost; 2241 if (stream_stats.width > info.send_frame_width) 2242 info.send_frame_width = stream_stats.width; 2243 if (stream_stats.height > info.send_frame_height) 2244 info.send_frame_height = stream_stats.height; 2245 info.firs_rcvd += stream_stats.rtcp_packet_type_counts.fir_packets; 2246 info.nacks_rcvd += stream_stats.rtcp_packet_type_counts.nack_packets; 2247 info.plis_rcvd += stream_stats.rtcp_packet_type_counts.pli_packets; 2248 } 2249 2250 if (!stats.substreams.empty()) { 2251 // TODO(pbos): Report fraction lost per SSRC. 2252 webrtc::VideoSendStream::StreamStats first_stream_stats = 2253 stats.substreams.begin()->second; 2254 info.fraction_lost = 2255 static_cast<float>(first_stream_stats.rtcp_stats.fraction_lost) / 2256 (1 << 8); 2257 } 2258 2259 return info; 2260} 2261 2262void WebRtcVideoChannel2::WebRtcVideoSendStream::FillBandwidthEstimationInfo( 2263 BandwidthEstimationInfo* bwe_info) { 2264 rtc::CritScope cs(&lock_); 2265 if (stream_ == NULL) { 2266 return; 2267 } 2268 webrtc::VideoSendStream::Stats stats = stream_->GetStats(); 2269 for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it = 2270 stats.substreams.begin(); 2271 it != stats.substreams.end(); ++it) { 2272 bwe_info->transmit_bitrate += it->second.total_bitrate_bps; 2273 bwe_info->retransmit_bitrate += it->second.retransmit_bitrate_bps; 2274 } 2275 bwe_info->target_enc_bitrate += stats.target_media_bitrate_bps; 2276 bwe_info->actual_enc_bitrate += stats.media_bitrate_bps; 2277} 2278 2279void WebRtcVideoChannel2::WebRtcVideoSendStream::SetMaxBitrateBps( 2280 int max_bitrate_bps) { 2281 rtc::CritScope cs(&lock_); 2282 parameters_.max_bitrate_bps = max_bitrate_bps; 2283 2284 // No need to reconfigure if the stream hasn't been configured yet. 2285 if (parameters_.encoder_config.streams.empty()) 2286 return; 2287 2288 // Force a stream reconfigure to set the new max bitrate. 2289 int width = last_dimensions_.width; 2290 last_dimensions_.width = 0; 2291 SetDimensions(width, last_dimensions_.height, last_dimensions_.is_screencast); 2292} 2293 2294void WebRtcVideoChannel2::WebRtcVideoSendStream::RecreateWebRtcStream() { 2295 if (stream_ != NULL) { 2296 call_->DestroyVideoSendStream(stream_); 2297 } 2298 2299 VideoCodecSettings codec_settings; 2300 parameters_.codec_settings.Get(&codec_settings); 2301 parameters_.encoder_config.encoder_specific_settings = 2302 ConfigureVideoEncoderSettings( 2303 codec_settings.codec, parameters_.options, 2304 parameters_.encoder_config.content_type == 2305 webrtc::VideoEncoderConfig::ContentType::kScreen); 2306 2307 webrtc::VideoSendStream::Config config = parameters_.config; 2308 if (!config.rtp.rtx.ssrcs.empty() && config.rtp.rtx.payload_type == -1) { 2309 LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX " 2310 "payload type the set codec. Ignoring RTX."; 2311 config.rtp.rtx.ssrcs.clear(); 2312 } 2313 stream_ = call_->CreateVideoSendStream(config, parameters_.encoder_config); 2314 2315 parameters_.encoder_config.encoder_specific_settings = NULL; 2316 2317 if (sending_) { 2318 stream_->Start(); 2319 } 2320} 2321 2322WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( 2323 webrtc::Call* call, 2324 const StreamParams& sp, 2325 const webrtc::VideoReceiveStream::Config& config, 2326 WebRtcVideoDecoderFactory* external_decoder_factory, 2327 bool default_stream, 2328 const std::vector<VideoCodecSettings>& recv_codecs) 2329 : call_(call), 2330 ssrcs_(sp.ssrcs), 2331 ssrc_groups_(sp.ssrc_groups), 2332 stream_(NULL), 2333 default_stream_(default_stream), 2334 config_(config), 2335 external_decoder_factory_(external_decoder_factory), 2336 renderer_(NULL), 2337 last_width_(-1), 2338 last_height_(-1), 2339 first_frame_timestamp_(-1), 2340 estimated_remote_start_ntp_time_ms_(0) { 2341 config_.renderer = this; 2342 // SetRecvCodecs will also reset (start) the VideoReceiveStream. 2343 LOG(LS_INFO) << "SetRecvCodecs (recv) because we are creating the receive " 2344 "stream for the first time: " 2345 << CodecSettingsVectorToString(recv_codecs); 2346 SetRecvCodecs(recv_codecs); 2347} 2348 2349WebRtcVideoChannel2::WebRtcVideoReceiveStream::AllocatedDecoder:: 2350 AllocatedDecoder(webrtc::VideoDecoder* decoder, 2351 webrtc::VideoCodecType type, 2352 bool external) 2353 : decoder(decoder), 2354 external_decoder(nullptr), 2355 type(type), 2356 external(external) { 2357 if (external) { 2358 external_decoder = decoder; 2359 this->decoder = 2360 new webrtc::VideoDecoderSoftwareFallbackWrapper(type, external_decoder); 2361 } 2362} 2363 2364WebRtcVideoChannel2::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() { 2365 call_->DestroyVideoReceiveStream(stream_); 2366 ClearDecoders(&allocated_decoders_); 2367} 2368 2369const std::vector<uint32_t>& 2370WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetSsrcs() const { 2371 return ssrcs_; 2372} 2373 2374WebRtcVideoChannel2::WebRtcVideoReceiveStream::AllocatedDecoder 2375WebRtcVideoChannel2::WebRtcVideoReceiveStream::CreateOrReuseVideoDecoder( 2376 std::vector<AllocatedDecoder>* old_decoders, 2377 const VideoCodec& codec) { 2378 webrtc::VideoCodecType type = CodecTypeFromName(codec.name); 2379 2380 for (size_t i = 0; i < old_decoders->size(); ++i) { 2381 if ((*old_decoders)[i].type == type) { 2382 AllocatedDecoder decoder = (*old_decoders)[i]; 2383 (*old_decoders)[i] = old_decoders->back(); 2384 old_decoders->pop_back(); 2385 return decoder; 2386 } 2387 } 2388 2389 if (external_decoder_factory_ != NULL) { 2390 webrtc::VideoDecoder* decoder = 2391 external_decoder_factory_->CreateVideoDecoder(type); 2392 if (decoder != NULL) { 2393 return AllocatedDecoder(decoder, type, true); 2394 } 2395 } 2396 2397 if (type == webrtc::kVideoCodecVP8) { 2398 return AllocatedDecoder( 2399 webrtc::VideoDecoder::Create(webrtc::VideoDecoder::kVp8), type, false); 2400 } 2401 2402 if (type == webrtc::kVideoCodecVP9) { 2403 return AllocatedDecoder( 2404 webrtc::VideoDecoder::Create(webrtc::VideoDecoder::kVp9), type, false); 2405 } 2406 2407 if (type == webrtc::kVideoCodecH264) { 2408 return AllocatedDecoder( 2409 webrtc::VideoDecoder::Create(webrtc::VideoDecoder::kH264), type, false); 2410 } 2411 2412 // This shouldn't happen, we should not be trying to create something we don't 2413 // support. 2414 RTC_DCHECK(false); 2415 return AllocatedDecoder(NULL, webrtc::kVideoCodecUnknown, false); 2416} 2417 2418void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRecvCodecs( 2419 const std::vector<VideoCodecSettings>& recv_codecs) { 2420 std::vector<AllocatedDecoder> old_decoders = allocated_decoders_; 2421 allocated_decoders_.clear(); 2422 config_.decoders.clear(); 2423 for (size_t i = 0; i < recv_codecs.size(); ++i) { 2424 AllocatedDecoder allocated_decoder = 2425 CreateOrReuseVideoDecoder(&old_decoders, recv_codecs[i].codec); 2426 allocated_decoders_.push_back(allocated_decoder); 2427 2428 webrtc::VideoReceiveStream::Decoder decoder; 2429 decoder.decoder = allocated_decoder.decoder; 2430 decoder.payload_type = recv_codecs[i].codec.id; 2431 decoder.payload_name = recv_codecs[i].codec.name; 2432 config_.decoders.push_back(decoder); 2433 } 2434 2435 // TODO(pbos): Reconfigure RTX based on incoming recv_codecs. 2436 config_.rtp.fec = recv_codecs.front().fec; 2437 config_.rtp.nack.rtp_history_ms = 2438 HasNack(recv_codecs.begin()->codec) ? kNackHistoryMs : 0; 2439 2440 ClearDecoders(&old_decoders); 2441 LOG(LS_INFO) << "RecreateWebRtcStream (recv) because of SetRecvCodecs: " 2442 << CodecSettingsVectorToString(recv_codecs); 2443 RecreateWebRtcStream(); 2444} 2445 2446void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetLocalSsrc( 2447 uint32_t local_ssrc) { 2448 // TODO(pbos): Consider turning this sanity check into a RTC_DCHECK. You 2449 // should not be able to create a sender with the same SSRC as a receiver, but 2450 // right now this can't be done due to unittests depending on receiving what 2451 // they are sending from the same MediaChannel. 2452 if (local_ssrc == config_.rtp.remote_ssrc) { 2453 LOG(LS_INFO) << "Ignoring call to SetLocalSsrc because parameters are " 2454 "unchanged; local_ssrc=" << local_ssrc; 2455 return; 2456 } 2457 2458 config_.rtp.local_ssrc = local_ssrc; 2459 LOG(LS_INFO) 2460 << "RecreateWebRtcStream (recv) because of SetLocalSsrc; local_ssrc=" 2461 << local_ssrc; 2462 RecreateWebRtcStream(); 2463} 2464 2465void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetNackAndRemb( 2466 bool nack_enabled, bool remb_enabled) { 2467 int nack_history_ms = nack_enabled ? kNackHistoryMs : 0; 2468 if (config_.rtp.nack.rtp_history_ms == nack_history_ms && 2469 config_.rtp.remb == remb_enabled) { 2470 LOG(LS_INFO) << "Ignoring call to SetNackAndRemb because parameters are " 2471 "unchanged; nack=" << nack_enabled 2472 << ", remb=" << remb_enabled; 2473 return; 2474 } 2475 config_.rtp.remb = remb_enabled; 2476 config_.rtp.nack.rtp_history_ms = nack_history_ms; 2477 LOG(LS_INFO) << "RecreateWebRtcStream (recv) because of SetNackAndRemb; nack=" 2478 << nack_enabled << ", remb=" << remb_enabled; 2479 RecreateWebRtcStream(); 2480} 2481 2482void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRtpExtensions( 2483 const std::vector<webrtc::RtpExtension>& extensions) { 2484 config_.rtp.extensions = extensions; 2485 LOG(LS_INFO) << "RecreateWebRtcStream (recv) because of SetRtpExtensions"; 2486 RecreateWebRtcStream(); 2487} 2488 2489void WebRtcVideoChannel2::WebRtcVideoReceiveStream::RecreateWebRtcStream() { 2490 if (stream_ != NULL) { 2491 call_->DestroyVideoReceiveStream(stream_); 2492 } 2493 stream_ = call_->CreateVideoReceiveStream(config_); 2494 stream_->Start(); 2495} 2496 2497void WebRtcVideoChannel2::WebRtcVideoReceiveStream::ClearDecoders( 2498 std::vector<AllocatedDecoder>* allocated_decoders) { 2499 for (size_t i = 0; i < allocated_decoders->size(); ++i) { 2500 if ((*allocated_decoders)[i].external) { 2501 external_decoder_factory_->DestroyVideoDecoder( 2502 (*allocated_decoders)[i].external_decoder); 2503 } 2504 delete (*allocated_decoders)[i].decoder; 2505 } 2506 allocated_decoders->clear(); 2507} 2508 2509void WebRtcVideoChannel2::WebRtcVideoReceiveStream::RenderFrame( 2510 const webrtc::VideoFrame& frame, 2511 int time_to_render_ms) { 2512 rtc::CritScope crit(&renderer_lock_); 2513 2514 if (first_frame_timestamp_ < 0) 2515 first_frame_timestamp_ = frame.timestamp(); 2516 int64_t rtp_time_elapsed_since_first_frame = 2517 (timestamp_wraparound_handler_.Unwrap(frame.timestamp()) - 2518 first_frame_timestamp_); 2519 int64_t elapsed_time_ms = rtp_time_elapsed_since_first_frame / 2520 (cricket::kVideoCodecClockrate / 1000); 2521 if (frame.ntp_time_ms() > 0) 2522 estimated_remote_start_ntp_time_ms_ = frame.ntp_time_ms() - elapsed_time_ms; 2523 2524 if (renderer_ == NULL) { 2525 LOG(LS_WARNING) << "VideoReceiveStream not connected to a VideoRenderer."; 2526 return; 2527 } 2528 2529 if (frame.width() != last_width_ || frame.height() != last_height_) { 2530 SetSize(frame.width(), frame.height()); 2531 } 2532 2533 const WebRtcVideoFrame render_frame( 2534 frame.video_frame_buffer(), 2535 frame.render_time_ms() * rtc::kNumNanosecsPerMillisec, frame.rotation()); 2536 renderer_->RenderFrame(&render_frame); 2537} 2538 2539bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::IsTextureSupported() const { 2540 return true; 2541} 2542 2543bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::IsDefaultStream() const { 2544 return default_stream_; 2545} 2546 2547void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRenderer( 2548 cricket::VideoRenderer* renderer) { 2549 rtc::CritScope crit(&renderer_lock_); 2550 renderer_ = renderer; 2551 if (renderer_ != NULL && last_width_ != -1) { 2552 SetSize(last_width_, last_height_); 2553 } 2554} 2555 2556VideoRenderer* WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetRenderer() { 2557 // TODO(pbos): Remove GetRenderer and all uses of it, it's thread-unsafe by 2558 // design. 2559 rtc::CritScope crit(&renderer_lock_); 2560 return renderer_; 2561} 2562 2563void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetSize(int width, 2564 int height) { 2565 rtc::CritScope crit(&renderer_lock_); 2566 if (!renderer_->SetSize(width, height, 0)) { 2567 LOG(LS_ERROR) << "Could not set renderer size."; 2568 } 2569 last_width_ = width; 2570 last_height_ = height; 2571} 2572 2573std::string 2574WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetCodecNameFromPayloadType( 2575 int payload_type) { 2576 for (const webrtc::VideoReceiveStream::Decoder& decoder : config_.decoders) { 2577 if (decoder.payload_type == payload_type) { 2578 return decoder.payload_name; 2579 } 2580 } 2581 return ""; 2582} 2583 2584VideoReceiverInfo 2585WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetVideoReceiverInfo() { 2586 VideoReceiverInfo info; 2587 info.ssrc_groups = ssrc_groups_; 2588 info.add_ssrc(config_.rtp.remote_ssrc); 2589 webrtc::VideoReceiveStream::Stats stats = stream_->GetStats(); 2590 info.bytes_rcvd = stats.rtp_stats.transmitted.payload_bytes + 2591 stats.rtp_stats.transmitted.header_bytes + 2592 stats.rtp_stats.transmitted.padding_bytes; 2593 info.packets_rcvd = stats.rtp_stats.transmitted.packets; 2594 info.packets_lost = stats.rtcp_stats.cumulative_lost; 2595 info.fraction_lost = 2596 static_cast<float>(stats.rtcp_stats.fraction_lost) / (1 << 8); 2597 2598 info.framerate_rcvd = stats.network_frame_rate; 2599 info.framerate_decoded = stats.decode_frame_rate; 2600 info.framerate_output = stats.render_frame_rate; 2601 2602 { 2603 rtc::CritScope frame_cs(&renderer_lock_); 2604 info.frame_width = last_width_; 2605 info.frame_height = last_height_; 2606 info.capture_start_ntp_time_ms = estimated_remote_start_ntp_time_ms_; 2607 } 2608 2609 info.decode_ms = stats.decode_ms; 2610 info.max_decode_ms = stats.max_decode_ms; 2611 info.current_delay_ms = stats.current_delay_ms; 2612 info.target_delay_ms = stats.target_delay_ms; 2613 info.jitter_buffer_ms = stats.jitter_buffer_ms; 2614 info.min_playout_delay_ms = stats.min_playout_delay_ms; 2615 info.render_delay_ms = stats.render_delay_ms; 2616 2617 info.codec_name = GetCodecNameFromPayloadType(stats.current_payload_type); 2618 2619 info.firs_sent = stats.rtcp_packet_type_counts.fir_packets; 2620 info.plis_sent = stats.rtcp_packet_type_counts.pli_packets; 2621 info.nacks_sent = stats.rtcp_packet_type_counts.nack_packets; 2622 2623 return info; 2624} 2625 2626WebRtcVideoChannel2::VideoCodecSettings::VideoCodecSettings() 2627 : rtx_payload_type(-1) {} 2628 2629bool WebRtcVideoChannel2::VideoCodecSettings::operator==( 2630 const WebRtcVideoChannel2::VideoCodecSettings& other) const { 2631 return codec == other.codec && 2632 fec.ulpfec_payload_type == other.fec.ulpfec_payload_type && 2633 fec.red_payload_type == other.fec.red_payload_type && 2634 fec.red_rtx_payload_type == other.fec.red_rtx_payload_type && 2635 rtx_payload_type == other.rtx_payload_type; 2636} 2637 2638bool WebRtcVideoChannel2::VideoCodecSettings::operator!=( 2639 const WebRtcVideoChannel2::VideoCodecSettings& other) const { 2640 return !(*this == other); 2641} 2642 2643std::vector<WebRtcVideoChannel2::VideoCodecSettings> 2644WebRtcVideoChannel2::MapCodecs(const std::vector<VideoCodec>& codecs) { 2645 RTC_DCHECK(!codecs.empty()); 2646 2647 std::vector<VideoCodecSettings> video_codecs; 2648 std::map<int, bool> payload_used; 2649 std::map<int, VideoCodec::CodecType> payload_codec_type; 2650 // |rtx_mapping| maps video payload type to rtx payload type. 2651 std::map<int, int> rtx_mapping; 2652 2653 webrtc::FecConfig fec_settings; 2654 2655 for (size_t i = 0; i < codecs.size(); ++i) { 2656 const VideoCodec& in_codec = codecs[i]; 2657 int payload_type = in_codec.id; 2658 2659 if (payload_used[payload_type]) { 2660 LOG(LS_ERROR) << "Payload type already registered: " 2661 << in_codec.ToString(); 2662 return std::vector<VideoCodecSettings>(); 2663 } 2664 payload_used[payload_type] = true; 2665 payload_codec_type[payload_type] = in_codec.GetCodecType(); 2666 2667 switch (in_codec.GetCodecType()) { 2668 case VideoCodec::CODEC_RED: { 2669 // RED payload type, should not have duplicates. 2670 RTC_DCHECK(fec_settings.red_payload_type == -1); 2671 fec_settings.red_payload_type = in_codec.id; 2672 continue; 2673 } 2674 2675 case VideoCodec::CODEC_ULPFEC: { 2676 // ULPFEC payload type, should not have duplicates. 2677 RTC_DCHECK(fec_settings.ulpfec_payload_type == -1); 2678 fec_settings.ulpfec_payload_type = in_codec.id; 2679 continue; 2680 } 2681 2682 case VideoCodec::CODEC_RTX: { 2683 int associated_payload_type; 2684 if (!in_codec.GetParam(kCodecParamAssociatedPayloadType, 2685 &associated_payload_type) || 2686 !IsValidRtpPayloadType(associated_payload_type)) { 2687 LOG(LS_ERROR) 2688 << "RTX codec with invalid or no associated payload type: " 2689 << in_codec.ToString(); 2690 return std::vector<VideoCodecSettings>(); 2691 } 2692 rtx_mapping[associated_payload_type] = in_codec.id; 2693 continue; 2694 } 2695 2696 case VideoCodec::CODEC_VIDEO: 2697 break; 2698 } 2699 2700 video_codecs.push_back(VideoCodecSettings()); 2701 video_codecs.back().codec = in_codec; 2702 } 2703 2704 // One of these codecs should have been a video codec. Only having FEC 2705 // parameters into this code is a logic error. 2706 RTC_DCHECK(!video_codecs.empty()); 2707 2708 for (std::map<int, int>::const_iterator it = rtx_mapping.begin(); 2709 it != rtx_mapping.end(); 2710 ++it) { 2711 if (!payload_used[it->first]) { 2712 LOG(LS_ERROR) << "RTX mapped to payload not in codec list."; 2713 return std::vector<VideoCodecSettings>(); 2714 } 2715 if (payload_codec_type[it->first] != VideoCodec::CODEC_VIDEO && 2716 payload_codec_type[it->first] != VideoCodec::CODEC_RED) { 2717 LOG(LS_ERROR) << "RTX not mapped to regular video codec or RED codec."; 2718 return std::vector<VideoCodecSettings>(); 2719 } 2720 2721 if (it->first == fec_settings.red_payload_type) { 2722 fec_settings.red_rtx_payload_type = it->second; 2723 } 2724 } 2725 2726 for (size_t i = 0; i < video_codecs.size(); ++i) { 2727 video_codecs[i].fec = fec_settings; 2728 if (rtx_mapping[video_codecs[i].codec.id] != 0 && 2729 rtx_mapping[video_codecs[i].codec.id] != 2730 fec_settings.red_payload_type) { 2731 video_codecs[i].rtx_payload_type = rtx_mapping[video_codecs[i].codec.id]; 2732 } 2733 } 2734 2735 return video_codecs; 2736} 2737 2738} // namespace cricket 2739 2740#endif // HAVE_WEBRTC_VIDEO 2741