webrtcvideoengine2.cc revision 9a4410e9934578e84cc129b978a29e151d957994
1/* 2 * libjingle 3 * Copyright 2014 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#ifdef HAVE_WEBRTC_VIDEO 29#include "talk/media/webrtc/webrtcvideoengine2.h" 30 31#include <algorithm> 32#include <set> 33#include <string> 34 35#include "libyuv/convert_from.h" 36#include "talk/media/base/videocapturer.h" 37#include "talk/media/base/videorenderer.h" 38#include "talk/media/webrtc/constants.h" 39#include "talk/media/webrtc/simulcast.h" 40#include "talk/media/webrtc/webrtcvideocapturer.h" 41#include "talk/media/webrtc/webrtcvideoengine.h" 42#include "talk/media/webrtc/webrtcvideoframe.h" 43#include "talk/media/webrtc/webrtcvoiceengine.h" 44#include "webrtc/base/buffer.h" 45#include "webrtc/base/logging.h" 46#include "webrtc/base/stringutils.h" 47#include "webrtc/call.h" 48#include "webrtc/system_wrappers/interface/trace_event.h" 49#include "webrtc/video_decoder.h" 50#include "webrtc/video_encoder.h" 51 52#define UNIMPLEMENTED \ 53 LOG(LS_ERROR) << "Call to unimplemented function " << __FUNCTION__; \ 54 ASSERT(false) 55 56namespace cricket { 57namespace { 58static std::string CodecVectorToString(const std::vector<VideoCodec>& codecs) { 59 std::stringstream out; 60 out << '{'; 61 for (size_t i = 0; i < codecs.size(); ++i) { 62 out << codecs[i].ToString(); 63 if (i != codecs.size() - 1) { 64 out << ", "; 65 } 66 } 67 out << '}'; 68 return out.str(); 69} 70 71static bool ValidateCodecFormats(const std::vector<VideoCodec>& codecs) { 72 bool has_video = false; 73 for (size_t i = 0; i < codecs.size(); ++i) { 74 if (!codecs[i].ValidateCodecFormat()) { 75 return false; 76 } 77 if (codecs[i].GetCodecType() == VideoCodec::CODEC_VIDEO) { 78 has_video = true; 79 } 80 } 81 if (!has_video) { 82 LOG(LS_ERROR) << "Setting codecs without a video codec is invalid: " 83 << CodecVectorToString(codecs); 84 return false; 85 } 86 return true; 87} 88 89static std::string RtpExtensionsToString( 90 const std::vector<RtpHeaderExtension>& extensions) { 91 std::stringstream out; 92 out << '{'; 93 for (size_t i = 0; i < extensions.size(); ++i) { 94 out << "{" << extensions[i].uri << ": " << extensions[i].id << "}"; 95 if (i != extensions.size() - 1) { 96 out << ", "; 97 } 98 } 99 out << '}'; 100 return out.str(); 101} 102 103// Merges two fec configs and logs an error if a conflict arises 104// such that merging in diferent order would trigger a diferent output. 105static void MergeFecConfig(const webrtc::FecConfig& other, 106 webrtc::FecConfig* output) { 107 if (other.ulpfec_payload_type != -1) { 108 if (output->ulpfec_payload_type != -1 && 109 output->ulpfec_payload_type != other.ulpfec_payload_type) { 110 LOG(LS_WARNING) << "Conflict merging ulpfec_payload_type configs: " 111 << output->ulpfec_payload_type << " and " 112 << other.ulpfec_payload_type; 113 } 114 output->ulpfec_payload_type = other.ulpfec_payload_type; 115 } 116 if (other.red_payload_type != -1) { 117 if (output->red_payload_type != -1 && 118 output->red_payload_type != other.red_payload_type) { 119 LOG(LS_WARNING) << "Conflict merging red_payload_type configs: " 120 << output->red_payload_type << " and " 121 << other.red_payload_type; 122 } 123 output->red_payload_type = other.red_payload_type; 124 } 125} 126} // namespace 127 128// This constant is really an on/off, lower-level configurable NACK history 129// duration hasn't been implemented. 130static const int kNackHistoryMs = 1000; 131 132static const int kDefaultQpMax = 56; 133 134static const int kDefaultRtcpReceiverReportSsrc = 1; 135 136const char kH264CodecName[] = "H264"; 137 138static bool FindFirstMatchingCodec(const std::vector<VideoCodec>& codecs, 139 const VideoCodec& requested_codec, 140 VideoCodec* matching_codec) { 141 for (size_t i = 0; i < codecs.size(); ++i) { 142 if (requested_codec.Matches(codecs[i])) { 143 *matching_codec = codecs[i]; 144 return true; 145 } 146 } 147 return false; 148} 149 150static bool ValidateRtpHeaderExtensionIds( 151 const std::vector<RtpHeaderExtension>& extensions) { 152 std::set<int> extensions_used; 153 for (size_t i = 0; i < extensions.size(); ++i) { 154 if (extensions[i].id < 0 || extensions[i].id >= 15 || 155 !extensions_used.insert(extensions[i].id).second) { 156 LOG(LS_ERROR) << "RTP extensions are with incorrect or duplicate ids."; 157 return false; 158 } 159 } 160 return true; 161} 162 163static bool CompareRtpHeaderExtensionIds( 164 const webrtc::RtpExtension& extension1, 165 const webrtc::RtpExtension& extension2) { 166 // Sorting on ID is sufficient, more than one extension per ID is unsupported. 167 return extension1.id > extension2.id; 168} 169 170static std::vector<webrtc::RtpExtension> FilterRtpExtensions( 171 const std::vector<RtpHeaderExtension>& extensions) { 172 std::vector<webrtc::RtpExtension> webrtc_extensions; 173 for (size_t i = 0; i < extensions.size(); ++i) { 174 // Unsupported extensions will be ignored. 175 if (webrtc::RtpExtension::IsSupported(extensions[i].uri)) { 176 webrtc_extensions.push_back(webrtc::RtpExtension( 177 extensions[i].uri, extensions[i].id)); 178 } else { 179 LOG(LS_WARNING) << "Unsupported RTP extension: " << extensions[i].uri; 180 } 181 } 182 183 // Sort filtered headers to make sure that they can later be compared 184 // regardless of in which order they were entered. 185 std::sort(webrtc_extensions.begin(), webrtc_extensions.end(), 186 CompareRtpHeaderExtensionIds); 187 return webrtc_extensions; 188} 189 190static bool RtpExtensionsHaveChanged( 191 const std::vector<webrtc::RtpExtension>& before, 192 const std::vector<webrtc::RtpExtension>& after) { 193 if (before.size() != after.size()) 194 return true; 195 for (size_t i = 0; i < before.size(); ++i) { 196 if (before[i].id != after[i].id) 197 return true; 198 if (before[i].name != after[i].name) 199 return true; 200 } 201 return false; 202} 203 204std::vector<webrtc::VideoStream> 205WebRtcVideoChannel2::WebRtcVideoSendStream::CreateSimulcastVideoStreams( 206 const VideoCodec& codec, 207 const VideoOptions& options, 208 size_t num_streams) { 209 // Use default factory for non-simulcast. 210 int max_qp = kDefaultQpMax; 211 codec.GetParam(kCodecParamMaxQuantization, &max_qp); 212 213 int min_bitrate_kbps; 214 if (!codec.GetParam(kCodecParamMinBitrate, &min_bitrate_kbps) || 215 min_bitrate_kbps < kMinVideoBitrate) { 216 min_bitrate_kbps = kMinVideoBitrate; 217 } 218 219 int max_bitrate_kbps; 220 if (!codec.GetParam(kCodecParamMaxBitrate, &max_bitrate_kbps)) { 221 max_bitrate_kbps = 0; 222 } 223 224 return GetSimulcastConfig( 225 num_streams, 226 GetSimulcastBitrateMode(options), 227 codec.width, 228 codec.height, 229 max_bitrate_kbps * 1000, 230 max_qp, 231 codec.framerate != 0 ? codec.framerate : kDefaultVideoMaxFramerate); 232} 233 234std::vector<webrtc::VideoStream> 235WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoStreams( 236 const VideoCodec& codec, 237 const VideoOptions& options, 238 size_t num_streams) { 239 if (num_streams != 1) 240 return CreateSimulcastVideoStreams(codec, options, num_streams); 241 242 webrtc::VideoStream stream; 243 stream.width = codec.width; 244 stream.height = codec.height; 245 stream.max_framerate = 246 codec.framerate != 0 ? codec.framerate : kDefaultVideoMaxFramerate; 247 248 stream.min_bitrate_bps = kMinVideoBitrate * 1000; 249 stream.target_bitrate_bps = stream.max_bitrate_bps = kMaxVideoBitrate * 1000; 250 251 int max_qp = kDefaultQpMax; 252 codec.GetParam(kCodecParamMaxQuantization, &max_qp); 253 stream.max_qp = max_qp; 254 std::vector<webrtc::VideoStream> streams; 255 streams.push_back(stream); 256 return streams; 257} 258 259void* WebRtcVideoChannel2::WebRtcVideoSendStream::ConfigureVideoEncoderSettings( 260 const VideoCodec& codec, 261 const VideoOptions& options) { 262 if (CodecNameMatches(codec.name, kVp8CodecName)) { 263 encoder_settings_.vp8 = webrtc::VideoEncoder::GetDefaultVp8Settings(); 264 options.video_noise_reduction.Get(&encoder_settings_.vp8.denoisingOn); 265 return &encoder_settings_.vp8; 266 } 267 if (CodecNameMatches(codec.name, kVp9CodecName)) { 268 encoder_settings_.vp9 = webrtc::VideoEncoder::GetDefaultVp9Settings(); 269 options.video_noise_reduction.Get(&encoder_settings_.vp9.denoisingOn); 270 return &encoder_settings_.vp9; 271 } 272 return NULL; 273} 274 275DefaultUnsignalledSsrcHandler::DefaultUnsignalledSsrcHandler() 276 : default_recv_ssrc_(0), default_renderer_(NULL) {} 277 278UnsignalledSsrcHandler::Action DefaultUnsignalledSsrcHandler::OnUnsignalledSsrc( 279 VideoMediaChannel* channel, 280 uint32_t ssrc) { 281 if (default_recv_ssrc_ != 0) { // Already one default stream. 282 LOG(LS_WARNING) << "Unknown SSRC, but default receive stream already set."; 283 return kDropPacket; 284 } 285 286 StreamParams sp; 287 sp.ssrcs.push_back(ssrc); 288 LOG(LS_INFO) << "Creating default receive stream for SSRC=" << ssrc << "."; 289 if (!channel->AddRecvStream(sp)) { 290 LOG(LS_WARNING) << "Could not create default receive stream."; 291 } 292 293 channel->SetRenderer(ssrc, default_renderer_); 294 default_recv_ssrc_ = ssrc; 295 return kDeliverPacket; 296} 297 298WebRtcCallFactory::~WebRtcCallFactory() { 299} 300webrtc::Call* WebRtcCallFactory::CreateCall( 301 const webrtc::Call::Config& config) { 302 return webrtc::Call::Create(config); 303} 304 305VideoRenderer* DefaultUnsignalledSsrcHandler::GetDefaultRenderer() const { 306 return default_renderer_; 307} 308 309void DefaultUnsignalledSsrcHandler::SetDefaultRenderer( 310 VideoMediaChannel* channel, 311 VideoRenderer* renderer) { 312 default_renderer_ = renderer; 313 if (default_recv_ssrc_ != 0) { 314 channel->SetRenderer(default_recv_ssrc_, default_renderer_); 315 } 316} 317 318WebRtcVideoEngine2::WebRtcVideoEngine2() 319 : worker_thread_(NULL), 320 voice_engine_(NULL), 321 default_codec_format_(kDefaultVideoMaxWidth, 322 kDefaultVideoMaxHeight, 323 FPS_TO_INTERVAL(kDefaultVideoMaxFramerate), 324 FOURCC_ANY), 325 initialized_(false), 326 call_factory_(&default_call_factory_), 327 external_decoder_factory_(NULL), 328 external_encoder_factory_(NULL) { 329 LOG(LS_INFO) << "WebRtcVideoEngine2::WebRtcVideoEngine2()"; 330 video_codecs_ = GetSupportedCodecs(); 331 rtp_header_extensions_.push_back( 332 RtpHeaderExtension(kRtpTimestampOffsetHeaderExtension, 333 kRtpTimestampOffsetHeaderExtensionDefaultId)); 334 rtp_header_extensions_.push_back( 335 RtpHeaderExtension(kRtpAbsoluteSenderTimeHeaderExtension, 336 kRtpAbsoluteSenderTimeHeaderExtensionDefaultId)); 337} 338 339WebRtcVideoEngine2::~WebRtcVideoEngine2() { 340 LOG(LS_INFO) << "WebRtcVideoEngine2::~WebRtcVideoEngine2"; 341 342 if (initialized_) { 343 Terminate(); 344 } 345} 346 347void WebRtcVideoEngine2::SetCallFactory(WebRtcCallFactory* call_factory) { 348 assert(!initialized_); 349 call_factory_ = call_factory; 350} 351 352bool WebRtcVideoEngine2::Init(rtc::Thread* worker_thread) { 353 LOG(LS_INFO) << "WebRtcVideoEngine2::Init"; 354 worker_thread_ = worker_thread; 355 ASSERT(worker_thread_ != NULL); 356 357 initialized_ = true; 358 return true; 359} 360 361void WebRtcVideoEngine2::Terminate() { 362 LOG(LS_INFO) << "WebRtcVideoEngine2::Terminate"; 363 364 initialized_ = false; 365} 366 367int WebRtcVideoEngine2::GetCapabilities() { return VIDEO_RECV | VIDEO_SEND; } 368 369bool WebRtcVideoEngine2::SetDefaultEncoderConfig( 370 const VideoEncoderConfig& config) { 371 const VideoCodec& codec = config.max_codec; 372 bool supports_codec = false; 373 for (size_t i = 0; i < video_codecs_.size(); ++i) { 374 if (CodecNameMatches(video_codecs_[i].name, codec.name)) { 375 video_codecs_[i] = codec; 376 supports_codec = true; 377 break; 378 } 379 } 380 381 if (!supports_codec) { 382 LOG(LS_ERROR) << "SetDefaultEncoderConfig, codec not supported: " 383 << codec.ToString(); 384 return false; 385 } 386 387 default_codec_format_ = 388 VideoFormat(codec.width, 389 codec.height, 390 VideoFormat::FpsToInterval(codec.framerate), 391 FOURCC_ANY); 392 return true; 393} 394 395WebRtcVideoChannel2* WebRtcVideoEngine2::CreateChannel( 396 const VideoOptions& options, 397 VoiceMediaChannel* voice_channel) { 398 assert(initialized_); 399 LOG(LS_INFO) << "CreateChannel: " 400 << (voice_channel != NULL ? "With" : "Without") 401 << " voice channel. Options: " << options.ToString(); 402 WebRtcVideoChannel2* channel = 403 new WebRtcVideoChannel2(call_factory_, 404 voice_engine_, 405 voice_channel, 406 options, 407 external_encoder_factory_, 408 external_decoder_factory_); 409 if (!channel->Init()) { 410 delete channel; 411 return NULL; 412 } 413 channel->SetRecvCodecs(video_codecs_); 414 return channel; 415} 416 417const std::vector<VideoCodec>& WebRtcVideoEngine2::codecs() const { 418 return video_codecs_; 419} 420 421const std::vector<RtpHeaderExtension>& 422WebRtcVideoEngine2::rtp_header_extensions() const { 423 return rtp_header_extensions_; 424} 425 426void WebRtcVideoEngine2::SetLogging(int min_sev, const char* filter) { 427 // TODO(pbos): Set up logging. 428 LOG(LS_VERBOSE) << "SetLogging: " << min_sev << '"' << filter << '"'; 429 // if min_sev == -1, we keep the current log level. 430 if (min_sev < 0) { 431 assert(min_sev == -1); 432 return; 433 } 434} 435 436void WebRtcVideoEngine2::SetExternalDecoderFactory( 437 WebRtcVideoDecoderFactory* decoder_factory) { 438 assert(!initialized_); 439 external_decoder_factory_ = decoder_factory; 440} 441 442void WebRtcVideoEngine2::SetExternalEncoderFactory( 443 WebRtcVideoEncoderFactory* encoder_factory) { 444 assert(!initialized_); 445 if (external_encoder_factory_ == encoder_factory) 446 return; 447 448 // No matter what happens we shouldn't hold on to a stale 449 // WebRtcSimulcastEncoderFactory. 450 simulcast_encoder_factory_.reset(); 451 452 if (encoder_factory && 453 WebRtcSimulcastEncoderFactory::UseSimulcastEncoderFactory( 454 encoder_factory->codecs())) { 455 simulcast_encoder_factory_.reset( 456 new WebRtcSimulcastEncoderFactory(encoder_factory)); 457 encoder_factory = simulcast_encoder_factory_.get(); 458 } 459 external_encoder_factory_ = encoder_factory; 460 461 video_codecs_ = GetSupportedCodecs(); 462} 463 464bool WebRtcVideoEngine2::EnableTimedRender() { 465 // TODO(pbos): Figure out whether this can be removed. 466 return true; 467} 468 469// Checks to see whether we comprehend and could receive a particular codec 470bool WebRtcVideoEngine2::FindCodec(const VideoCodec& in) { 471 // TODO(pbos): Probe encoder factory to figure out that the codec is supported 472 // if supported by the encoder factory. Add a corresponding test that fails 473 // with this code (that doesn't ask the factory). 474 for (size_t j = 0; j < video_codecs_.size(); ++j) { 475 VideoCodec codec(video_codecs_[j].id, video_codecs_[j].name, 0, 0, 0, 0); 476 if (codec.Matches(in)) { 477 return true; 478 } 479 } 480 return false; 481} 482 483// Tells whether the |requested| codec can be transmitted or not. If it can be 484// transmitted |out| is set with the best settings supported. Aspect ratio will 485// be set as close to |current|'s as possible. If not set |requested|'s 486// dimensions will be used for aspect ratio matching. 487bool WebRtcVideoEngine2::CanSendCodec(const VideoCodec& requested, 488 const VideoCodec& current, 489 VideoCodec* out) { 490 assert(out != NULL); 491 492 if (requested.width != requested.height && 493 (requested.height == 0 || requested.width == 0)) { 494 // 0xn and nx0 are invalid resolutions. 495 return false; 496 } 497 498 VideoCodec matching_codec; 499 if (!FindFirstMatchingCodec(video_codecs_, requested, &matching_codec)) { 500 // Codec not supported. 501 return false; 502 } 503 504 out->id = requested.id; 505 out->name = requested.name; 506 out->preference = requested.preference; 507 out->params = requested.params; 508 out->framerate = std::min(requested.framerate, matching_codec.framerate); 509 out->params = requested.params; 510 out->feedback_params = requested.feedback_params; 511 out->width = requested.width; 512 out->height = requested.height; 513 if (requested.width == 0 && requested.height == 0) { 514 return true; 515 } 516 517 while (out->width > matching_codec.width) { 518 out->width /= 2; 519 out->height /= 2; 520 } 521 522 return out->width > 0 && out->height > 0; 523} 524 525bool WebRtcVideoEngine2::SetVoiceEngine(WebRtcVoiceEngine* voice_engine) { 526 if (initialized_) { 527 LOG(LS_WARNING) << "SetVoiceEngine can not be called after Init"; 528 return false; 529 } 530 voice_engine_ = voice_engine; 531 return true; 532} 533 534// Ignore spammy trace messages, mostly from the stats API when we haven't 535// gotten RTCP info yet from the remote side. 536bool WebRtcVideoEngine2::ShouldIgnoreTrace(const std::string& trace) { 537 static const char* const kTracesToIgnore[] = {NULL}; 538 for (const char* const* p = kTracesToIgnore; *p; ++p) { 539 if (trace.find(*p) == 0) { 540 return true; 541 } 542 } 543 return false; 544} 545 546std::vector<VideoCodec> WebRtcVideoEngine2::GetSupportedCodecs() const { 547 std::vector<VideoCodec> supported_codecs = DefaultVideoCodecList(); 548 549 if (external_encoder_factory_ == NULL) { 550 return supported_codecs; 551 } 552 553 const std::vector<WebRtcVideoEncoderFactory::VideoCodec>& codecs = 554 external_encoder_factory_->codecs(); 555 for (size_t i = 0; i < codecs.size(); ++i) { 556 // Don't add internally-supported codecs twice. 557 if (CodecIsInternallySupported(codecs[i].name)) { 558 continue; 559 } 560 561 // External video encoders are given payloads 120-127. This also means that 562 // we only support up to 8 external payload types. 563 const int kExternalVideoPayloadTypeBase = 120; 564 size_t payload_type = kExternalVideoPayloadTypeBase + i; 565 assert(payload_type < 128); 566 VideoCodec codec(static_cast<int>(payload_type), 567 codecs[i].name, 568 codecs[i].max_width, 569 codecs[i].max_height, 570 codecs[i].max_fps, 571 0); 572 573 AddDefaultFeedbackParams(&codec); 574 supported_codecs.push_back(codec); 575 } 576 return supported_codecs; 577} 578 579WebRtcVideoChannel2::WebRtcVideoChannel2( 580 WebRtcCallFactory* call_factory, 581 WebRtcVoiceEngine* voice_engine, 582 VoiceMediaChannel* voice_channel, 583 const VideoOptions& options, 584 WebRtcVideoEncoderFactory* external_encoder_factory, 585 WebRtcVideoDecoderFactory* external_decoder_factory) 586 : unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_), 587 voice_channel_(voice_channel), 588 external_encoder_factory_(external_encoder_factory), 589 external_decoder_factory_(external_decoder_factory) { 590 SetDefaultOptions(); 591 options_.SetAll(options); 592 webrtc::Call::Config config(this); 593 config.overuse_callback = this; 594 if (voice_engine != NULL) { 595 config.voice_engine = voice_engine->voe()->engine(); 596 } 597 598 call_.reset(call_factory->CreateCall(config)); 599 600 rtcp_receiver_report_ssrc_ = kDefaultRtcpReceiverReportSsrc; 601 sending_ = false; 602 default_send_ssrc_ = 0; 603} 604 605void WebRtcVideoChannel2::SetDefaultOptions() { 606 options_.cpu_overuse_detection.Set(false); 607 options_.dscp.Set(false); 608 options_.suspend_below_min_bitrate.Set(false); 609 options_.video_noise_reduction.Set(true); 610 options_.screencast_min_bitrate.Set(0); 611} 612 613WebRtcVideoChannel2::~WebRtcVideoChannel2() { 614 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 615 send_streams_.begin(); 616 it != send_streams_.end(); 617 ++it) { 618 delete it->second; 619 } 620 621 for (std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 622 receive_streams_.begin(); 623 it != receive_streams_.end(); 624 ++it) { 625 delete it->second; 626 } 627} 628 629bool WebRtcVideoChannel2::Init() { return true; } 630 631bool WebRtcVideoChannel2::CodecIsExternallySupported( 632 const std::string& name) const { 633 if (external_encoder_factory_ == NULL) { 634 return false; 635 } 636 637 const std::vector<WebRtcVideoEncoderFactory::VideoCodec> external_codecs = 638 external_encoder_factory_->codecs(); 639 for (size_t c = 0; c < external_codecs.size(); ++c) { 640 if (CodecNameMatches(name, external_codecs[c].name)) { 641 return true; 642 } 643 } 644 return false; 645} 646 647std::vector<WebRtcVideoChannel2::VideoCodecSettings> 648WebRtcVideoChannel2::FilterSupportedCodecs( 649 const std::vector<WebRtcVideoChannel2::VideoCodecSettings>& mapped_codecs) 650 const { 651 std::vector<VideoCodecSettings> supported_codecs; 652 for (size_t i = 0; i < mapped_codecs.size(); ++i) { 653 const VideoCodecSettings& codec = mapped_codecs[i]; 654 if (CodecIsInternallySupported(codec.codec.name) || 655 CodecIsExternallySupported(codec.codec.name)) { 656 supported_codecs.push_back(codec); 657 } 658 } 659 return supported_codecs; 660} 661 662bool WebRtcVideoChannel2::SetRecvCodecs(const std::vector<VideoCodec>& codecs) { 663 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetRecvCodecs"); 664 LOG(LS_INFO) << "SetRecvCodecs: " << CodecVectorToString(codecs); 665 if (!ValidateCodecFormats(codecs)) { 666 return false; 667 } 668 669 const std::vector<VideoCodecSettings> mapped_codecs = MapCodecs(codecs); 670 if (mapped_codecs.empty()) { 671 LOG(LS_ERROR) << "SetRecvCodecs called without any video codecs."; 672 return false; 673 } 674 675 const std::vector<VideoCodecSettings> supported_codecs = 676 FilterSupportedCodecs(mapped_codecs); 677 678 if (mapped_codecs.size() != supported_codecs.size()) { 679 LOG(LS_ERROR) << "SetRecvCodecs called with unsupported video codecs."; 680 return false; 681 } 682 683 recv_codecs_ = supported_codecs; 684 685 rtc::CritScope stream_lock(&stream_crit_); 686 for (std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 687 receive_streams_.begin(); 688 it != receive_streams_.end(); 689 ++it) { 690 it->second->SetRecvCodecs(recv_codecs_); 691 } 692 693 return true; 694} 695 696bool WebRtcVideoChannel2::SetSendCodecs(const std::vector<VideoCodec>& codecs) { 697 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetSendCodecs"); 698 LOG(LS_INFO) << "SetSendCodecs: " << CodecVectorToString(codecs); 699 if (!ValidateCodecFormats(codecs)) { 700 return false; 701 } 702 703 const std::vector<VideoCodecSettings> supported_codecs = 704 FilterSupportedCodecs(MapCodecs(codecs)); 705 706 if (supported_codecs.empty()) { 707 LOG(LS_ERROR) << "No video codecs supported by encoder factory."; 708 return false; 709 } 710 711 LOG(LS_INFO) << "Using codec: " << supported_codecs.front().codec.ToString(); 712 713 VideoCodecSettings old_codec; 714 if (send_codec_.Get(&old_codec) && supported_codecs.front() == old_codec) { 715 // Using same codec, avoid reconfiguring. 716 return true; 717 } 718 719 send_codec_.Set(supported_codecs.front()); 720 721 rtc::CritScope stream_lock(&stream_crit_); 722 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 723 send_streams_.begin(); 724 it != send_streams_.end(); 725 ++it) { 726 assert(it->second != NULL); 727 it->second->SetCodec(supported_codecs.front()); 728 } 729 730 VideoCodec codec = supported_codecs.front().codec; 731 int bitrate_kbps; 732 if (codec.GetParam(kCodecParamMinBitrate, &bitrate_kbps) && 733 bitrate_kbps > 0) { 734 bitrate_config_.min_bitrate_bps = bitrate_kbps * 1000; 735 } else { 736 bitrate_config_.min_bitrate_bps = 0; 737 } 738 if (codec.GetParam(kCodecParamStartBitrate, &bitrate_kbps) && 739 bitrate_kbps > 0) { 740 bitrate_config_.start_bitrate_bps = bitrate_kbps * 1000; 741 } else { 742 // Do not reconfigure start bitrate unless it's specified and positive. 743 bitrate_config_.start_bitrate_bps = -1; 744 } 745 if (codec.GetParam(kCodecParamMaxBitrate, &bitrate_kbps) && 746 bitrate_kbps > 0) { 747 bitrate_config_.max_bitrate_bps = bitrate_kbps * 1000; 748 } else { 749 bitrate_config_.max_bitrate_bps = -1; 750 } 751 call_->SetBitrateConfig(bitrate_config_); 752 753 return true; 754} 755 756bool WebRtcVideoChannel2::GetSendCodec(VideoCodec* codec) { 757 VideoCodecSettings codec_settings; 758 if (!send_codec_.Get(&codec_settings)) { 759 LOG(LS_VERBOSE) << "GetSendCodec: No send codec set."; 760 return false; 761 } 762 *codec = codec_settings.codec; 763 return true; 764} 765 766bool WebRtcVideoChannel2::SetSendStreamFormat(uint32 ssrc, 767 const VideoFormat& format) { 768 LOG(LS_VERBOSE) << "SetSendStreamFormat:" << ssrc << " -> " 769 << format.ToString(); 770 rtc::CritScope stream_lock(&stream_crit_); 771 if (send_streams_.find(ssrc) == send_streams_.end()) { 772 return false; 773 } 774 return send_streams_[ssrc]->SetVideoFormat(format); 775} 776 777bool WebRtcVideoChannel2::SetRender(bool render) { 778 // TODO(pbos): Implement. Or refactor away as it shouldn't be needed. 779 LOG(LS_VERBOSE) << "SetRender: " << (render ? "true" : "false"); 780 return true; 781} 782 783bool WebRtcVideoChannel2::SetSend(bool send) { 784 LOG(LS_VERBOSE) << "SetSend: " << (send ? "true" : "false"); 785 if (send && !send_codec_.IsSet()) { 786 LOG(LS_ERROR) << "SetSend(true) called before setting codec."; 787 return false; 788 } 789 if (send) { 790 StartAllSendStreams(); 791 } else { 792 StopAllSendStreams(); 793 } 794 sending_ = send; 795 return true; 796} 797 798bool WebRtcVideoChannel2::AddSendStream(const StreamParams& sp) { 799 LOG(LS_INFO) << "AddSendStream: " << sp.ToString(); 800 if (sp.ssrcs.empty()) { 801 LOG(LS_ERROR) << "No SSRCs in stream parameters."; 802 return false; 803 } 804 805 uint32 ssrc = sp.first_ssrc(); 806 assert(ssrc != 0); 807 // TODO(pbos): Make sure none of sp.ssrcs are used, not just the identifying 808 // ssrc. 809 rtc::CritScope stream_lock(&stream_crit_); 810 if (send_streams_.find(ssrc) != send_streams_.end()) { 811 LOG(LS_ERROR) << "Send stream with ssrc '" << ssrc << "' already exists."; 812 return false; 813 } 814 815 std::vector<uint32> primary_ssrcs; 816 sp.GetPrimarySsrcs(&primary_ssrcs); 817 std::vector<uint32> rtx_ssrcs; 818 sp.GetFidSsrcs(primary_ssrcs, &rtx_ssrcs); 819 if (!rtx_ssrcs.empty() && primary_ssrcs.size() != rtx_ssrcs.size()) { 820 LOG(LS_ERROR) 821 << "RTX SSRCs exist, but don't cover all SSRCs (unsupported): " 822 << sp.ToString(); 823 return false; 824 } 825 826 WebRtcVideoSendStream* stream = 827 new WebRtcVideoSendStream(call_.get(), 828 external_encoder_factory_, 829 options_, 830 send_codec_, 831 sp, 832 send_rtp_extensions_); 833 834 send_streams_[ssrc] = stream; 835 836 if (rtcp_receiver_report_ssrc_ == kDefaultRtcpReceiverReportSsrc) { 837 rtcp_receiver_report_ssrc_ = ssrc; 838 } 839 if (default_send_ssrc_ == 0) { 840 default_send_ssrc_ = ssrc; 841 } 842 if (sending_) { 843 stream->Start(); 844 } 845 846 return true; 847} 848 849bool WebRtcVideoChannel2::RemoveSendStream(uint32 ssrc) { 850 LOG(LS_INFO) << "RemoveSendStream: " << ssrc; 851 852 if (ssrc == 0) { 853 if (default_send_ssrc_ == 0) { 854 LOG(LS_ERROR) << "No default send stream active."; 855 return false; 856 } 857 858 LOG(LS_VERBOSE) << "Removing default stream: " << default_send_ssrc_; 859 ssrc = default_send_ssrc_; 860 } 861 862 WebRtcVideoSendStream* removed_stream; 863 { 864 rtc::CritScope stream_lock(&stream_crit_); 865 std::map<uint32, WebRtcVideoSendStream*>::iterator it = 866 send_streams_.find(ssrc); 867 if (it == send_streams_.end()) { 868 return false; 869 } 870 871 removed_stream = it->second; 872 send_streams_.erase(it); 873 } 874 875 delete removed_stream; 876 877 if (ssrc == default_send_ssrc_) { 878 default_send_ssrc_ = 0; 879 } 880 881 return true; 882} 883 884bool WebRtcVideoChannel2::AddRecvStream(const StreamParams& sp) { 885 LOG(LS_INFO) << "AddRecvStream: " << sp.ToString(); 886 assert(sp.ssrcs.size() > 0); 887 888 uint32 ssrc = sp.first_ssrc(); 889 assert(ssrc != 0); // TODO(pbos): Is this ever valid? 890 891 // TODO(pbos): Check if any of the SSRCs overlap. 892 rtc::CritScope stream_lock(&stream_crit_); 893 if (receive_streams_.find(ssrc) != receive_streams_.end()) { 894 LOG(LS_ERROR) << "Receive stream for SSRC " << ssrc << "already exists."; 895 return false; 896 } 897 898 webrtc::VideoReceiveStream::Config config; 899 ConfigureReceiverRtp(&config, sp); 900 901 // Set up A/V sync if there is a VoiceChannel. 902 // TODO(pbos): The A/V is synched by the receiving channel. So we need to know 903 // the SSRC of the remote audio channel in order to sync the correct webrtc 904 // VoiceEngine channel. For now sync the first channel in non-conference to 905 // match existing behavior in WebRtcVideoEngine. 906 if (voice_channel_ != NULL && receive_streams_.empty() && 907 !options_.conference_mode.GetWithDefaultIfUnset(false)) { 908 config.audio_channel_id = 909 static_cast<WebRtcVoiceMediaChannel*>(voice_channel_)->voe_channel(); 910 } 911 912 receive_streams_[ssrc] = new WebRtcVideoReceiveStream( 913 call_.get(), external_decoder_factory_, config, recv_codecs_); 914 915 return true; 916} 917 918void WebRtcVideoChannel2::ConfigureReceiverRtp( 919 webrtc::VideoReceiveStream::Config* config, 920 const StreamParams& sp) const { 921 uint32 ssrc = sp.first_ssrc(); 922 923 config->rtp.remote_ssrc = ssrc; 924 config->rtp.local_ssrc = rtcp_receiver_report_ssrc_; 925 926 config->rtp.extensions = recv_rtp_extensions_; 927 928 // TODO(pbos): This protection is against setting the same local ssrc as 929 // remote which is not permitted by the lower-level API. RTCP requires a 930 // corresponding sender SSRC. Figure out what to do when we don't have 931 // (receive-only) or know a good local SSRC. 932 if (config->rtp.remote_ssrc == config->rtp.local_ssrc) { 933 if (config->rtp.local_ssrc != kDefaultRtcpReceiverReportSsrc) { 934 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc; 935 } else { 936 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc + 1; 937 } 938 } 939 940 for (size_t i = 0; i < recv_codecs_.size(); ++i) { 941 MergeFecConfig(recv_codecs_[i].fec, &config->rtp.fec); 942 } 943 944 for (size_t i = 0; i < recv_codecs_.size(); ++i) { 945 uint32 rtx_ssrc; 946 if (recv_codecs_[i].rtx_payload_type != -1 && 947 sp.GetFidSsrc(ssrc, &rtx_ssrc)) { 948 webrtc::VideoReceiveStream::Config::Rtp::Rtx& rtx = 949 config->rtp.rtx[recv_codecs_[i].codec.id]; 950 rtx.ssrc = rtx_ssrc; 951 rtx.payload_type = recv_codecs_[i].rtx_payload_type; 952 } 953 } 954} 955 956bool WebRtcVideoChannel2::RemoveRecvStream(uint32 ssrc) { 957 LOG(LS_INFO) << "RemoveRecvStream: " << ssrc; 958 if (ssrc == 0) { 959 LOG(LS_ERROR) << "RemoveRecvStream with 0 ssrc is not supported."; 960 return false; 961 } 962 963 rtc::CritScope stream_lock(&stream_crit_); 964 std::map<uint32, WebRtcVideoReceiveStream*>::iterator stream = 965 receive_streams_.find(ssrc); 966 if (stream == receive_streams_.end()) { 967 LOG(LS_ERROR) << "Stream not found for ssrc: " << ssrc; 968 return false; 969 } 970 delete stream->second; 971 receive_streams_.erase(stream); 972 973 return true; 974} 975 976bool WebRtcVideoChannel2::SetRenderer(uint32 ssrc, VideoRenderer* renderer) { 977 LOG(LS_INFO) << "SetRenderer: ssrc:" << ssrc << " " 978 << (renderer ? "(ptr)" : "NULL"); 979 if (ssrc == 0) { 980 default_unsignalled_ssrc_handler_.SetDefaultRenderer(this, renderer); 981 return true; 982 } 983 984 rtc::CritScope stream_lock(&stream_crit_); 985 std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 986 receive_streams_.find(ssrc); 987 if (it == receive_streams_.end()) { 988 return false; 989 } 990 991 it->second->SetRenderer(renderer); 992 return true; 993} 994 995bool WebRtcVideoChannel2::GetRenderer(uint32 ssrc, VideoRenderer** renderer) { 996 if (ssrc == 0) { 997 *renderer = default_unsignalled_ssrc_handler_.GetDefaultRenderer(); 998 return *renderer != NULL; 999 } 1000 1001 rtc::CritScope stream_lock(&stream_crit_); 1002 std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 1003 receive_streams_.find(ssrc); 1004 if (it == receive_streams_.end()) { 1005 return false; 1006 } 1007 *renderer = it->second->GetRenderer(); 1008 return true; 1009} 1010 1011bool WebRtcVideoChannel2::GetStats(const StatsOptions& options, 1012 VideoMediaInfo* info) { 1013 info->Clear(); 1014 FillSenderStats(info); 1015 FillReceiverStats(info); 1016 webrtc::Call::Stats stats = call_->GetStats(); 1017 FillBandwidthEstimationStats(stats, info); 1018 if (stats.rtt_ms != -1) { 1019 for (size_t i = 0; i < info->senders.size(); ++i) { 1020 info->senders[i].rtt_ms = stats.rtt_ms; 1021 } 1022 } 1023 return true; 1024} 1025 1026void WebRtcVideoChannel2::FillSenderStats(VideoMediaInfo* video_media_info) { 1027 rtc::CritScope stream_lock(&stream_crit_); 1028 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1029 send_streams_.begin(); 1030 it != send_streams_.end(); 1031 ++it) { 1032 video_media_info->senders.push_back(it->second->GetVideoSenderInfo()); 1033 } 1034} 1035 1036void WebRtcVideoChannel2::FillReceiverStats(VideoMediaInfo* video_media_info) { 1037 rtc::CritScope stream_lock(&stream_crit_); 1038 for (std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 1039 receive_streams_.begin(); 1040 it != receive_streams_.end(); 1041 ++it) { 1042 video_media_info->receivers.push_back(it->second->GetVideoReceiverInfo()); 1043 } 1044} 1045 1046void WebRtcVideoChannel2::FillBandwidthEstimationStats( 1047 const webrtc::Call::Stats& stats, 1048 VideoMediaInfo* video_media_info) { 1049 BandwidthEstimationInfo bwe_info; 1050 bwe_info.available_send_bandwidth = stats.send_bandwidth_bps; 1051 bwe_info.available_recv_bandwidth = stats.recv_bandwidth_bps; 1052 bwe_info.bucket_delay = stats.pacer_delay_ms; 1053 1054 // Get send stream bitrate stats. 1055 rtc::CritScope stream_lock(&stream_crit_); 1056 for (std::map<uint32, WebRtcVideoSendStream*>::iterator stream = 1057 send_streams_.begin(); 1058 stream != send_streams_.end(); 1059 ++stream) { 1060 stream->second->FillBandwidthEstimationInfo(&bwe_info); 1061 } 1062 video_media_info->bw_estimations.push_back(bwe_info); 1063} 1064 1065bool WebRtcVideoChannel2::SetCapturer(uint32 ssrc, VideoCapturer* capturer) { 1066 LOG(LS_INFO) << "SetCapturer: " << ssrc << " -> " 1067 << (capturer != NULL ? "(capturer)" : "NULL"); 1068 assert(ssrc != 0); 1069 rtc::CritScope stream_lock(&stream_crit_); 1070 if (send_streams_.find(ssrc) == send_streams_.end()) { 1071 LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc; 1072 return false; 1073 } 1074 return send_streams_[ssrc]->SetCapturer(capturer); 1075} 1076 1077bool WebRtcVideoChannel2::SendIntraFrame() { 1078 // TODO(pbos): Implement. 1079 LOG(LS_VERBOSE) << "SendIntraFrame()."; 1080 return true; 1081} 1082 1083bool WebRtcVideoChannel2::RequestIntraFrame() { 1084 // TODO(pbos): Implement. 1085 LOG(LS_VERBOSE) << "SendIntraFrame()."; 1086 return true; 1087} 1088 1089void WebRtcVideoChannel2::OnPacketReceived( 1090 rtc::Buffer* packet, 1091 const rtc::PacketTime& packet_time) { 1092 const webrtc::PacketReceiver::DeliveryStatus delivery_result = 1093 call_->Receiver()->DeliverPacket( 1094 reinterpret_cast<const uint8_t*>(packet->data()), packet->length()); 1095 switch (delivery_result) { 1096 case webrtc::PacketReceiver::DELIVERY_OK: 1097 return; 1098 case webrtc::PacketReceiver::DELIVERY_PACKET_ERROR: 1099 return; 1100 case webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC: 1101 break; 1102 } 1103 1104 uint32 ssrc = 0; 1105 if (!GetRtpSsrc(packet->data(), packet->length(), &ssrc)) { 1106 return; 1107 } 1108 1109 // TODO(pbos): Make sure that the unsignalled SSRC uses the video payload. 1110 // Also figure out whether RTX needs to be handled. 1111 switch (unsignalled_ssrc_handler_->OnUnsignalledSsrc(this, ssrc)) { 1112 case UnsignalledSsrcHandler::kDropPacket: 1113 return; 1114 case UnsignalledSsrcHandler::kDeliverPacket: 1115 break; 1116 } 1117 1118 if (call_->Receiver()->DeliverPacket( 1119 reinterpret_cast<const uint8_t*>(packet->data()), packet->length()) != 1120 webrtc::PacketReceiver::DELIVERY_OK) { 1121 LOG(LS_WARNING) << "Failed to deliver RTP packet on re-delivery."; 1122 return; 1123 } 1124} 1125 1126void WebRtcVideoChannel2::OnRtcpReceived( 1127 rtc::Buffer* packet, 1128 const rtc::PacketTime& packet_time) { 1129 if (call_->Receiver()->DeliverPacket( 1130 reinterpret_cast<const uint8_t*>(packet->data()), packet->length()) != 1131 webrtc::PacketReceiver::DELIVERY_OK) { 1132 LOG(LS_WARNING) << "Failed to deliver RTCP packet."; 1133 } 1134} 1135 1136void WebRtcVideoChannel2::OnReadyToSend(bool ready) { 1137 LOG(LS_VERBOSE) << "OnReadyToSend: " << (ready ? "Ready." : "Not ready."); 1138 call_->SignalNetworkState(ready ? webrtc::Call::kNetworkUp 1139 : webrtc::Call::kNetworkDown); 1140} 1141 1142bool WebRtcVideoChannel2::MuteStream(uint32 ssrc, bool mute) { 1143 LOG(LS_VERBOSE) << "MuteStream: " << ssrc << " -> " 1144 << (mute ? "mute" : "unmute"); 1145 assert(ssrc != 0); 1146 rtc::CritScope stream_lock(&stream_crit_); 1147 if (send_streams_.find(ssrc) == send_streams_.end()) { 1148 LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc; 1149 return false; 1150 } 1151 1152 send_streams_[ssrc]->MuteStream(mute); 1153 return true; 1154} 1155 1156bool WebRtcVideoChannel2::SetRecvRtpHeaderExtensions( 1157 const std::vector<RtpHeaderExtension>& extensions) { 1158 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetRecvRtpHeaderExtensions"); 1159 LOG(LS_INFO) << "SetRecvRtpHeaderExtensions: " 1160 << RtpExtensionsToString(extensions); 1161 if (!ValidateRtpHeaderExtensionIds(extensions)) 1162 return false; 1163 1164 std::vector<webrtc::RtpExtension> filtered_extensions = 1165 FilterRtpExtensions(extensions); 1166 if (!RtpExtensionsHaveChanged(recv_rtp_extensions_, filtered_extensions)) 1167 return true; 1168 1169 recv_rtp_extensions_ = filtered_extensions; 1170 1171 rtc::CritScope stream_lock(&stream_crit_); 1172 for (std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 1173 receive_streams_.begin(); 1174 it != receive_streams_.end(); 1175 ++it) { 1176 it->second->SetRtpExtensions(recv_rtp_extensions_); 1177 } 1178 return true; 1179} 1180 1181bool WebRtcVideoChannel2::SetSendRtpHeaderExtensions( 1182 const std::vector<RtpHeaderExtension>& extensions) { 1183 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetSendRtpHeaderExtensions"); 1184 LOG(LS_INFO) << "SetSendRtpHeaderExtensions: " 1185 << RtpExtensionsToString(extensions); 1186 if (!ValidateRtpHeaderExtensionIds(extensions)) 1187 return false; 1188 1189 std::vector<webrtc::RtpExtension> filtered_extensions = 1190 FilterRtpExtensions(extensions); 1191 if (!RtpExtensionsHaveChanged(send_rtp_extensions_, filtered_extensions)) 1192 return true; 1193 1194 send_rtp_extensions_ = filtered_extensions; 1195 1196 rtc::CritScope stream_lock(&stream_crit_); 1197 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1198 send_streams_.begin(); 1199 it != send_streams_.end(); 1200 ++it) { 1201 it->second->SetRtpExtensions(send_rtp_extensions_); 1202 } 1203 return true; 1204} 1205 1206bool WebRtcVideoChannel2::SetMaxSendBandwidth(int max_bitrate_bps) { 1207 LOG(LS_INFO) << "SetMaxSendBandwidth: " << max_bitrate_bps << "bps."; 1208 if (max_bitrate_bps <= 0) { 1209 // Unsetting max bitrate. 1210 max_bitrate_bps = -1; 1211 } 1212 bitrate_config_.start_bitrate_bps = -1; 1213 bitrate_config_.max_bitrate_bps = max_bitrate_bps; 1214 if (max_bitrate_bps > 0 && 1215 bitrate_config_.min_bitrate_bps > max_bitrate_bps) { 1216 bitrate_config_.min_bitrate_bps = max_bitrate_bps; 1217 } 1218 call_->SetBitrateConfig(bitrate_config_); 1219 return true; 1220} 1221 1222bool WebRtcVideoChannel2::SetOptions(const VideoOptions& options) { 1223 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetOptions"); 1224 LOG(LS_INFO) << "SetOptions: " << options.ToString(); 1225 VideoOptions old_options = options_; 1226 options_.SetAll(options); 1227 if (options_ == old_options) { 1228 // No new options to set. 1229 return true; 1230 } 1231 rtc::DiffServCodePoint dscp = options_.dscp.GetWithDefaultIfUnset(false) 1232 ? rtc::DSCP_AF41 1233 : rtc::DSCP_DEFAULT; 1234 MediaChannel::SetDscp(dscp); 1235 rtc::CritScope stream_lock(&stream_crit_); 1236 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1237 send_streams_.begin(); 1238 it != send_streams_.end(); 1239 ++it) { 1240 it->second->SetOptions(options_); 1241 } 1242 return true; 1243} 1244 1245void WebRtcVideoChannel2::SetInterface(NetworkInterface* iface) { 1246 MediaChannel::SetInterface(iface); 1247 // Set the RTP recv/send buffer to a bigger size 1248 MediaChannel::SetOption(NetworkInterface::ST_RTP, 1249 rtc::Socket::OPT_RCVBUF, 1250 kVideoRtpBufferSize); 1251 1252 // Speculative change to increase the outbound socket buffer size. 1253 // In b/15152257, we are seeing a significant number of packets discarded 1254 // due to lack of socket buffer space, although it's not yet clear what the 1255 // ideal value should be. 1256 MediaChannel::SetOption(NetworkInterface::ST_RTP, 1257 rtc::Socket::OPT_SNDBUF, 1258 kVideoRtpBufferSize); 1259} 1260 1261void WebRtcVideoChannel2::UpdateAspectRatio(int ratio_w, int ratio_h) { 1262 // TODO(pbos): Implement. 1263} 1264 1265void WebRtcVideoChannel2::OnMessage(rtc::Message* msg) { 1266 // Ignored. 1267} 1268 1269void WebRtcVideoChannel2::OnLoadUpdate(Load load) { 1270 rtc::CritScope stream_lock(&stream_crit_); 1271 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1272 send_streams_.begin(); 1273 it != send_streams_.end(); 1274 ++it) { 1275 it->second->OnCpuResolutionRequest(load == kOveruse 1276 ? CoordinatedVideoAdapter::DOWNGRADE 1277 : CoordinatedVideoAdapter::UPGRADE); 1278 } 1279} 1280 1281bool WebRtcVideoChannel2::SendRtp(const uint8_t* data, size_t len) { 1282 rtc::Buffer packet(data, len, kMaxRtpPacketLen); 1283 return MediaChannel::SendPacket(&packet); 1284} 1285 1286bool WebRtcVideoChannel2::SendRtcp(const uint8_t* data, size_t len) { 1287 rtc::Buffer packet(data, len, kMaxRtpPacketLen); 1288 return MediaChannel::SendRtcp(&packet); 1289} 1290 1291void WebRtcVideoChannel2::StartAllSendStreams() { 1292 rtc::CritScope stream_lock(&stream_crit_); 1293 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1294 send_streams_.begin(); 1295 it != send_streams_.end(); 1296 ++it) { 1297 it->second->Start(); 1298 } 1299} 1300 1301void WebRtcVideoChannel2::StopAllSendStreams() { 1302 rtc::CritScope stream_lock(&stream_crit_); 1303 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1304 send_streams_.begin(); 1305 it != send_streams_.end(); 1306 ++it) { 1307 it->second->Stop(); 1308 } 1309} 1310 1311WebRtcVideoChannel2::WebRtcVideoSendStream::VideoSendStreamParameters:: 1312 VideoSendStreamParameters( 1313 const webrtc::VideoSendStream::Config& config, 1314 const VideoOptions& options, 1315 const Settable<VideoCodecSettings>& codec_settings) 1316 : config(config), options(options), codec_settings(codec_settings) { 1317} 1318 1319WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream( 1320 webrtc::Call* call, 1321 WebRtcVideoEncoderFactory* external_encoder_factory, 1322 const VideoOptions& options, 1323 const Settable<VideoCodecSettings>& codec_settings, 1324 const StreamParams& sp, 1325 const std::vector<webrtc::RtpExtension>& rtp_extensions) 1326 : call_(call), 1327 external_encoder_factory_(external_encoder_factory), 1328 stream_(NULL), 1329 parameters_(webrtc::VideoSendStream::Config(), options, codec_settings), 1330 allocated_encoder_(NULL, webrtc::kVideoCodecUnknown, false), 1331 capturer_(NULL), 1332 sending_(false), 1333 muted_(false), 1334 old_adapt_changes_(0) { 1335 parameters_.config.rtp.max_packet_size = kVideoMtu; 1336 1337 sp.GetPrimarySsrcs(¶meters_.config.rtp.ssrcs); 1338 sp.GetFidSsrcs(parameters_.config.rtp.ssrcs, 1339 ¶meters_.config.rtp.rtx.ssrcs); 1340 parameters_.config.rtp.c_name = sp.cname; 1341 parameters_.config.rtp.extensions = rtp_extensions; 1342 1343 VideoCodecSettings params; 1344 if (codec_settings.Get(¶ms)) { 1345 SetCodec(params); 1346 } 1347} 1348 1349WebRtcVideoChannel2::WebRtcVideoSendStream::~WebRtcVideoSendStream() { 1350 DisconnectCapturer(); 1351 if (stream_ != NULL) { 1352 call_->DestroyVideoSendStream(stream_); 1353 } 1354 DestroyVideoEncoder(&allocated_encoder_); 1355} 1356 1357static void CreateBlackFrame(webrtc::I420VideoFrame* video_frame, 1358 int width, 1359 int height) { 1360 video_frame->CreateEmptyFrame(width, height, width, (width + 1) / 2, 1361 (width + 1) / 2); 1362 memset(video_frame->buffer(webrtc::kYPlane), 16, 1363 video_frame->allocated_size(webrtc::kYPlane)); 1364 memset(video_frame->buffer(webrtc::kUPlane), 128, 1365 video_frame->allocated_size(webrtc::kUPlane)); 1366 memset(video_frame->buffer(webrtc::kVPlane), 128, 1367 video_frame->allocated_size(webrtc::kVPlane)); 1368} 1369 1370static void ConvertToI420VideoFrame(const VideoFrame& frame, 1371 webrtc::I420VideoFrame* i420_frame) { 1372 i420_frame->CreateFrame( 1373 static_cast<int>(frame.GetYPitch() * frame.GetHeight()), 1374 frame.GetYPlane(), 1375 static_cast<int>(frame.GetUPitch() * ((frame.GetHeight() + 1) / 2)), 1376 frame.GetUPlane(), 1377 static_cast<int>(frame.GetVPitch() * ((frame.GetHeight() + 1) / 2)), 1378 frame.GetVPlane(), 1379 static_cast<int>(frame.GetWidth()), 1380 static_cast<int>(frame.GetHeight()), 1381 static_cast<int>(frame.GetYPitch()), 1382 static_cast<int>(frame.GetUPitch()), 1383 static_cast<int>(frame.GetVPitch())); 1384} 1385 1386void WebRtcVideoChannel2::WebRtcVideoSendStream::InputFrame( 1387 VideoCapturer* capturer, 1388 const VideoFrame* frame) { 1389 TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::InputFrame"); 1390 LOG(LS_VERBOSE) << "InputFrame: " << frame->GetWidth() << "x" 1391 << frame->GetHeight(); 1392 // Lock before copying, can be called concurrently when swapping input source. 1393 rtc::CritScope frame_cs(&frame_lock_); 1394 ConvertToI420VideoFrame(*frame, &video_frame_); 1395 1396 rtc::CritScope cs(&lock_); 1397 if (stream_ == NULL) { 1398 LOG(LS_WARNING) << "Capturer inputting frames before send codecs are " 1399 "configured, dropping."; 1400 return; 1401 } 1402 1403 // Not sending, abort early to prevent expensive reconfigurations while 1404 // setting up codecs etc. 1405 if (!sending_) 1406 return; 1407 1408 if (format_.width == 0) { // Dropping frames. 1409 assert(format_.height == 0); 1410 LOG(LS_VERBOSE) << "VideoFormat 0x0 set, Dropping frame."; 1411 return; 1412 } 1413 if (muted_) { 1414 // Create a black frame to transmit instead. 1415 CreateBlackFrame(&video_frame_, 1416 static_cast<int>(frame->GetWidth()), 1417 static_cast<int>(frame->GetHeight())); 1418 } 1419 // Reconfigure codec if necessary. 1420 SetDimensions( 1421 video_frame_.width(), video_frame_.height(), capturer->IsScreencast()); 1422 1423 LOG(LS_VERBOSE) << "SwapFrame: " << video_frame_.width() << "x" 1424 << video_frame_.height() << " -> (codec) " 1425 << parameters_.encoder_config.streams.back().width << "x" 1426 << parameters_.encoder_config.streams.back().height; 1427 stream_->Input()->SwapFrame(&video_frame_); 1428} 1429 1430bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetCapturer( 1431 VideoCapturer* capturer) { 1432 TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetCapturer"); 1433 if (!DisconnectCapturer() && capturer == NULL) { 1434 return false; 1435 } 1436 1437 { 1438 rtc::CritScope cs(&lock_); 1439 1440 if (capturer == NULL) { 1441 if (stream_ != NULL) { 1442 LOG(LS_VERBOSE) << "Disabling capturer, sending black frame."; 1443 webrtc::I420VideoFrame black_frame; 1444 1445 CreateBlackFrame(&black_frame, last_dimensions_.width, 1446 last_dimensions_.height); 1447 stream_->Input()->SwapFrame(&black_frame); 1448 } 1449 1450 capturer_ = NULL; 1451 return true; 1452 } 1453 1454 capturer_ = capturer; 1455 } 1456 // Lock cannot be held while connecting the capturer to prevent lock-order 1457 // violations. 1458 capturer->SignalVideoFrame.connect(this, &WebRtcVideoSendStream::InputFrame); 1459 return true; 1460} 1461 1462bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetVideoFormat( 1463 const VideoFormat& format) { 1464 if ((format.width == 0 || format.height == 0) && 1465 format.width != format.height) { 1466 LOG(LS_ERROR) << "Can't set VideoFormat, width or height is zero (but not " 1467 "both, 0x0 drops frames)."; 1468 return false; 1469 } 1470 1471 rtc::CritScope cs(&lock_); 1472 if (format.width == 0 && format.height == 0) { 1473 LOG(LS_INFO) 1474 << "0x0 resolution selected. Captured frames will be dropped for ssrc: " 1475 << parameters_.config.rtp.ssrcs[0] << "."; 1476 } else { 1477 // TODO(pbos): Fix me, this only affects the last stream! 1478 parameters_.encoder_config.streams.back().max_framerate = 1479 VideoFormat::IntervalToFps(format.interval); 1480 SetDimensions(format.width, format.height, false); 1481 } 1482 1483 format_ = format; 1484 return true; 1485} 1486 1487void WebRtcVideoChannel2::WebRtcVideoSendStream::MuteStream(bool mute) { 1488 rtc::CritScope cs(&lock_); 1489 muted_ = mute; 1490} 1491 1492bool WebRtcVideoChannel2::WebRtcVideoSendStream::DisconnectCapturer() { 1493 cricket::VideoCapturer* capturer; 1494 { 1495 rtc::CritScope cs(&lock_); 1496 if (capturer_ == NULL) 1497 return false; 1498 1499 if (capturer_->video_adapter() != nullptr) 1500 old_adapt_changes_ += capturer_->video_adapter()->adaptation_changes(); 1501 1502 capturer = capturer_; 1503 capturer_ = NULL; 1504 } 1505 capturer->SignalVideoFrame.disconnect(this); 1506 return true; 1507} 1508 1509void WebRtcVideoChannel2::WebRtcVideoSendStream::SetOptions( 1510 const VideoOptions& options) { 1511 rtc::CritScope cs(&lock_); 1512 VideoCodecSettings codec_settings; 1513 if (parameters_.codec_settings.Get(&codec_settings)) { 1514 SetCodecAndOptions(codec_settings, options); 1515 } else { 1516 parameters_.options = options; 1517 } 1518} 1519 1520void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodec( 1521 const VideoCodecSettings& codec_settings) { 1522 rtc::CritScope cs(&lock_); 1523 SetCodecAndOptions(codec_settings, parameters_.options); 1524} 1525 1526webrtc::VideoCodecType CodecTypeFromName(const std::string& name) { 1527 if (CodecNameMatches(name, kVp8CodecName)) { 1528 return webrtc::kVideoCodecVP8; 1529 } else if (CodecNameMatches(name, kVp9CodecName)) { 1530 return webrtc::kVideoCodecVP9; 1531 } else if (CodecNameMatches(name, kH264CodecName)) { 1532 return webrtc::kVideoCodecH264; 1533 } 1534 return webrtc::kVideoCodecUnknown; 1535} 1536 1537WebRtcVideoChannel2::WebRtcVideoSendStream::AllocatedEncoder 1538WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoder( 1539 const VideoCodec& codec) { 1540 webrtc::VideoCodecType type = CodecTypeFromName(codec.name); 1541 1542 // Do not re-create encoders of the same type. 1543 if (type == allocated_encoder_.type && allocated_encoder_.encoder != NULL) { 1544 return allocated_encoder_; 1545 } 1546 1547 if (external_encoder_factory_ != NULL) { 1548 webrtc::VideoEncoder* encoder = 1549 external_encoder_factory_->CreateVideoEncoder(type); 1550 if (encoder != NULL) { 1551 return AllocatedEncoder(encoder, type, true); 1552 } 1553 } 1554 1555 if (type == webrtc::kVideoCodecVP8) { 1556 return AllocatedEncoder( 1557 webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kVp8), type, false); 1558 } else if (type == webrtc::kVideoCodecVP9) { 1559 return AllocatedEncoder( 1560 webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kVp9), type, false); 1561 } 1562 1563 // This shouldn't happen, we should not be trying to create something we don't 1564 // support. 1565 assert(false); 1566 return AllocatedEncoder(NULL, webrtc::kVideoCodecUnknown, false); 1567} 1568 1569void WebRtcVideoChannel2::WebRtcVideoSendStream::DestroyVideoEncoder( 1570 AllocatedEncoder* encoder) { 1571 if (encoder->external) { 1572 external_encoder_factory_->DestroyVideoEncoder(encoder->encoder); 1573 } else { 1574 delete encoder->encoder; 1575 } 1576} 1577 1578void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodecAndOptions( 1579 const VideoCodecSettings& codec_settings, 1580 const VideoOptions& options) { 1581 parameters_.encoder_config = 1582 CreateVideoEncoderConfig(last_dimensions_, codec_settings.codec); 1583 if (parameters_.encoder_config.streams.empty()) 1584 return; 1585 1586 format_ = VideoFormat(codec_settings.codec.width, 1587 codec_settings.codec.height, 1588 VideoFormat::FpsToInterval(30), 1589 FOURCC_I420); 1590 1591 AllocatedEncoder new_encoder = CreateVideoEncoder(codec_settings.codec); 1592 parameters_.config.encoder_settings.encoder = new_encoder.encoder; 1593 parameters_.config.encoder_settings.payload_name = codec_settings.codec.name; 1594 parameters_.config.encoder_settings.payload_type = codec_settings.codec.id; 1595 parameters_.config.rtp.fec = codec_settings.fec; 1596 1597 // Set RTX payload type if RTX is enabled. 1598 if (!parameters_.config.rtp.rtx.ssrcs.empty()) { 1599 parameters_.config.rtp.rtx.payload_type = codec_settings.rtx_payload_type; 1600 } 1601 1602 if (IsNackEnabled(codec_settings.codec)) { 1603 parameters_.config.rtp.nack.rtp_history_ms = kNackHistoryMs; 1604 } 1605 1606 options.suspend_below_min_bitrate.Get( 1607 ¶meters_.config.suspend_below_min_bitrate); 1608 1609 parameters_.codec_settings.Set(codec_settings); 1610 parameters_.options = options; 1611 1612 RecreateWebRtcStream(); 1613 if (allocated_encoder_.encoder != new_encoder.encoder) { 1614 DestroyVideoEncoder(&allocated_encoder_); 1615 allocated_encoder_ = new_encoder; 1616 } 1617} 1618 1619void WebRtcVideoChannel2::WebRtcVideoSendStream::SetRtpExtensions( 1620 const std::vector<webrtc::RtpExtension>& rtp_extensions) { 1621 rtc::CritScope cs(&lock_); 1622 parameters_.config.rtp.extensions = rtp_extensions; 1623 RecreateWebRtcStream(); 1624} 1625 1626webrtc::VideoEncoderConfig 1627WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoderConfig( 1628 const Dimensions& dimensions, 1629 const VideoCodec& codec) const { 1630 webrtc::VideoEncoderConfig encoder_config; 1631 if (dimensions.is_screencast) { 1632 int screencast_min_bitrate_kbps; 1633 parameters_.options.screencast_min_bitrate.Get( 1634 &screencast_min_bitrate_kbps); 1635 encoder_config.min_transmit_bitrate_bps = 1636 screencast_min_bitrate_kbps * 1000; 1637 encoder_config.content_type = webrtc::VideoEncoderConfig::kScreenshare; 1638 } else { 1639 encoder_config.min_transmit_bitrate_bps = 0; 1640 encoder_config.content_type = webrtc::VideoEncoderConfig::kRealtimeVideo; 1641 } 1642 1643 // Restrict dimensions according to codec max. 1644 int width = dimensions.width; 1645 int height = dimensions.height; 1646 if (!dimensions.is_screencast) { 1647 if (codec.width < width) 1648 width = codec.width; 1649 if (codec.height < height) 1650 height = codec.height; 1651 } 1652 1653 VideoCodec clamped_codec = codec; 1654 clamped_codec.width = width; 1655 clamped_codec.height = height; 1656 1657 encoder_config.streams = CreateVideoStreams( 1658 clamped_codec, parameters_.options, parameters_.config.rtp.ssrcs.size()); 1659 1660 // Conference mode screencast uses 2 temporal layers split at 100kbit. 1661 if (parameters_.options.conference_mode.GetWithDefaultIfUnset(false) && 1662 dimensions.is_screencast && encoder_config.streams.size() == 1) { 1663 ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault(); 1664 1665 // For screenshare in conference mode, tl0 and tl1 bitrates are piggybacked 1666 // on the VideoCodec struct as target and max bitrates, respectively. 1667 // See eg. webrtc::VP8EncoderImpl::SetRates(). 1668 encoder_config.streams[0].target_bitrate_bps = 1669 config.tl0_bitrate_kbps * 1000; 1670 encoder_config.streams[0].max_bitrate_bps = config.tl1_bitrate_kbps * 1000; 1671 encoder_config.streams[0].temporal_layer_thresholds_bps.clear(); 1672 encoder_config.streams[0].temporal_layer_thresholds_bps.push_back( 1673 config.tl0_bitrate_kbps * 1000); 1674 } 1675 return encoder_config; 1676} 1677 1678void WebRtcVideoChannel2::WebRtcVideoSendStream::SetDimensions( 1679 int width, 1680 int height, 1681 bool is_screencast) { 1682 if (last_dimensions_.width == width && last_dimensions_.height == height && 1683 last_dimensions_.is_screencast == is_screencast) { 1684 // Configured using the same parameters, do not reconfigure. 1685 return; 1686 } 1687 LOG(LS_INFO) << "SetDimensions: " << width << "x" << height 1688 << (is_screencast ? " (screencast)" : " (not screencast)"); 1689 1690 last_dimensions_.width = width; 1691 last_dimensions_.height = height; 1692 last_dimensions_.is_screencast = is_screencast; 1693 1694 assert(!parameters_.encoder_config.streams.empty()); 1695 1696 VideoCodecSettings codec_settings; 1697 parameters_.codec_settings.Get(&codec_settings); 1698 1699 webrtc::VideoEncoderConfig encoder_config = 1700 CreateVideoEncoderConfig(last_dimensions_, codec_settings.codec); 1701 1702 encoder_config.encoder_specific_settings = 1703 ConfigureVideoEncoderSettings(codec_settings.codec, parameters_.options); 1704 1705 bool stream_reconfigured = stream_->ReconfigureVideoEncoder(encoder_config); 1706 1707 encoder_config.encoder_specific_settings = NULL; 1708 1709 if (!stream_reconfigured) { 1710 LOG(LS_WARNING) << "Failed to reconfigure video encoder for dimensions: " 1711 << width << "x" << height; 1712 return; 1713 } 1714 1715 parameters_.encoder_config = encoder_config; 1716} 1717 1718void WebRtcVideoChannel2::WebRtcVideoSendStream::Start() { 1719 rtc::CritScope cs(&lock_); 1720 assert(stream_ != NULL); 1721 stream_->Start(); 1722 sending_ = true; 1723} 1724 1725void WebRtcVideoChannel2::WebRtcVideoSendStream::Stop() { 1726 rtc::CritScope cs(&lock_); 1727 if (stream_ != NULL) { 1728 stream_->Stop(); 1729 } 1730 sending_ = false; 1731} 1732 1733VideoSenderInfo 1734WebRtcVideoChannel2::WebRtcVideoSendStream::GetVideoSenderInfo() { 1735 VideoSenderInfo info; 1736 webrtc::VideoSendStream::Stats stats; 1737 { 1738 rtc::CritScope cs(&lock_); 1739 for (uint32_t ssrc : parameters_.config.rtp.ssrcs) 1740 info.add_ssrc(ssrc); 1741 1742 for (size_t i = 0; i < parameters_.encoder_config.streams.size(); ++i) { 1743 if (i == parameters_.encoder_config.streams.size() - 1) { 1744 info.preferred_bitrate += 1745 parameters_.encoder_config.streams[i].max_bitrate_bps; 1746 } else { 1747 info.preferred_bitrate += 1748 parameters_.encoder_config.streams[i].target_bitrate_bps; 1749 } 1750 } 1751 1752 if (stream_ == NULL) 1753 return info; 1754 1755 stats = stream_->GetStats(); 1756 1757 info.adapt_changes = old_adapt_changes_; 1758 info.adapt_reason = CoordinatedVideoAdapter::ADAPTREASON_NONE; 1759 1760 if (capturer_ != NULL) { 1761 if (!capturer_->IsMuted()) { 1762 VideoFormat last_captured_frame_format; 1763 capturer_->GetStats(&info.adapt_frame_drops, &info.effects_frame_drops, 1764 &info.capturer_frame_time, 1765 &last_captured_frame_format); 1766 info.input_frame_width = last_captured_frame_format.width; 1767 info.input_frame_height = last_captured_frame_format.height; 1768 } 1769 if (capturer_->video_adapter() != nullptr) { 1770 info.adapt_changes += capturer_->video_adapter()->adaptation_changes(); 1771 info.adapt_reason = capturer_->video_adapter()->adapt_reason(); 1772 } 1773 } 1774 } 1775 info.framerate_input = stats.input_frame_rate; 1776 info.framerate_sent = stats.encode_frame_rate; 1777 1778 info.nominal_bitrate = stats.media_bitrate_bps; 1779 1780 info.send_frame_width = 0; 1781 info.send_frame_height = 0; 1782 for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it = 1783 stats.substreams.begin(); 1784 it != stats.substreams.end(); ++it) { 1785 // TODO(pbos): Wire up additional stats, such as padding bytes. 1786 webrtc::VideoSendStream::StreamStats stream_stats = it->second; 1787 info.bytes_sent += stream_stats.rtp_stats.transmitted.payload_bytes + 1788 stream_stats.rtp_stats.transmitted.header_bytes + 1789 stream_stats.rtp_stats.transmitted.padding_bytes; 1790 info.packets_sent += stream_stats.rtp_stats.transmitted.packets; 1791 info.packets_lost += stream_stats.rtcp_stats.cumulative_lost; 1792 if (stream_stats.width > info.send_frame_width) 1793 info.send_frame_width = stream_stats.width; 1794 if (stream_stats.height > info.send_frame_height) 1795 info.send_frame_height = stream_stats.height; 1796 info.firs_rcvd += stream_stats.rtcp_packet_type_counts.fir_packets; 1797 info.nacks_rcvd += stream_stats.rtcp_packet_type_counts.nack_packets; 1798 info.plis_rcvd += stream_stats.rtcp_packet_type_counts.pli_packets; 1799 } 1800 1801 if (!stats.substreams.empty()) { 1802 // TODO(pbos): Report fraction lost per SSRC. 1803 webrtc::VideoSendStream::StreamStats first_stream_stats = 1804 stats.substreams.begin()->second; 1805 info.fraction_lost = 1806 static_cast<float>(first_stream_stats.rtcp_stats.fraction_lost) / 1807 (1 << 8); 1808 } 1809 1810 return info; 1811} 1812 1813void WebRtcVideoChannel2::WebRtcVideoSendStream::FillBandwidthEstimationInfo( 1814 BandwidthEstimationInfo* bwe_info) { 1815 rtc::CritScope cs(&lock_); 1816 if (stream_ == NULL) { 1817 return; 1818 } 1819 webrtc::VideoSendStream::Stats stats = stream_->GetStats(); 1820 for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it = 1821 stats.substreams.begin(); 1822 it != stats.substreams.end(); ++it) { 1823 bwe_info->transmit_bitrate += it->second.total_bitrate_bps; 1824 bwe_info->retransmit_bitrate += it->second.retransmit_bitrate_bps; 1825 } 1826 bwe_info->actual_enc_bitrate += stats.media_bitrate_bps; 1827} 1828 1829void WebRtcVideoChannel2::WebRtcVideoSendStream::OnCpuResolutionRequest( 1830 CoordinatedVideoAdapter::AdaptRequest adapt_request) { 1831 rtc::CritScope cs(&lock_); 1832 bool adapt_cpu; 1833 parameters_.options.cpu_overuse_detection.Get(&adapt_cpu); 1834 if (!adapt_cpu) 1835 return; 1836 if (capturer_ == NULL || capturer_->video_adapter() == NULL) 1837 return; 1838 1839 capturer_->video_adapter()->OnCpuResolutionRequest(adapt_request); 1840} 1841 1842void WebRtcVideoChannel2::WebRtcVideoSendStream::RecreateWebRtcStream() { 1843 if (stream_ != NULL) { 1844 call_->DestroyVideoSendStream(stream_); 1845 } 1846 1847 VideoCodecSettings codec_settings; 1848 parameters_.codec_settings.Get(&codec_settings); 1849 parameters_.encoder_config.encoder_specific_settings = 1850 ConfigureVideoEncoderSettings(codec_settings.codec, parameters_.options); 1851 1852 stream_ = call_->CreateVideoSendStream(parameters_.config, 1853 parameters_.encoder_config); 1854 1855 parameters_.encoder_config.encoder_specific_settings = NULL; 1856 1857 if (sending_) { 1858 stream_->Start(); 1859 } 1860} 1861 1862WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( 1863 webrtc::Call* call, 1864 WebRtcVideoDecoderFactory* external_decoder_factory, 1865 const webrtc::VideoReceiveStream::Config& config, 1866 const std::vector<VideoCodecSettings>& recv_codecs) 1867 : call_(call), 1868 stream_(NULL), 1869 config_(config), 1870 external_decoder_factory_(external_decoder_factory), 1871 renderer_(NULL), 1872 last_width_(-1), 1873 last_height_(-1), 1874 first_frame_timestamp_(-1), 1875 estimated_remote_start_ntp_time_ms_(0) { 1876 config_.renderer = this; 1877 // SetRecvCodecs will also reset (start) the VideoReceiveStream. 1878 SetRecvCodecs(recv_codecs); 1879} 1880 1881WebRtcVideoChannel2::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() { 1882 call_->DestroyVideoReceiveStream(stream_); 1883 ClearDecoders(&allocated_decoders_); 1884} 1885 1886WebRtcVideoChannel2::WebRtcVideoReceiveStream::AllocatedDecoder 1887WebRtcVideoChannel2::WebRtcVideoReceiveStream::CreateOrReuseVideoDecoder( 1888 std::vector<AllocatedDecoder>* old_decoders, 1889 const VideoCodec& codec) { 1890 webrtc::VideoCodecType type = CodecTypeFromName(codec.name); 1891 1892 for (size_t i = 0; i < old_decoders->size(); ++i) { 1893 if ((*old_decoders)[i].type == type) { 1894 AllocatedDecoder decoder = (*old_decoders)[i]; 1895 (*old_decoders)[i] = old_decoders->back(); 1896 old_decoders->pop_back(); 1897 return decoder; 1898 } 1899 } 1900 1901 if (external_decoder_factory_ != NULL) { 1902 webrtc::VideoDecoder* decoder = 1903 external_decoder_factory_->CreateVideoDecoder(type); 1904 if (decoder != NULL) { 1905 return AllocatedDecoder(decoder, type, true); 1906 } 1907 } 1908 1909 if (type == webrtc::kVideoCodecVP8) { 1910 return AllocatedDecoder( 1911 webrtc::VideoDecoder::Create(webrtc::VideoDecoder::kVp8), type, false); 1912 } 1913 1914 // This shouldn't happen, we should not be trying to create something we don't 1915 // support. 1916 assert(false); 1917 return AllocatedDecoder(NULL, webrtc::kVideoCodecUnknown, false); 1918} 1919 1920void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRecvCodecs( 1921 const std::vector<VideoCodecSettings>& recv_codecs) { 1922 std::vector<AllocatedDecoder> old_decoders = allocated_decoders_; 1923 allocated_decoders_.clear(); 1924 config_.decoders.clear(); 1925 for (size_t i = 0; i < recv_codecs.size(); ++i) { 1926 AllocatedDecoder allocated_decoder = 1927 CreateOrReuseVideoDecoder(&old_decoders, recv_codecs[i].codec); 1928 allocated_decoders_.push_back(allocated_decoder); 1929 1930 webrtc::VideoReceiveStream::Decoder decoder; 1931 decoder.decoder = allocated_decoder.decoder; 1932 decoder.payload_type = recv_codecs[i].codec.id; 1933 decoder.payload_name = recv_codecs[i].codec.name; 1934 config_.decoders.push_back(decoder); 1935 } 1936 1937 // TODO(pbos): Reconfigure RTX based on incoming recv_codecs. 1938 config_.rtp.fec = recv_codecs.front().fec; 1939 config_.rtp.nack.rtp_history_ms = 1940 IsNackEnabled(recv_codecs.begin()->codec) ? kNackHistoryMs : 0; 1941 config_.rtp.remb = IsRembEnabled(recv_codecs.begin()->codec); 1942 1943 ClearDecoders(&old_decoders); 1944 RecreateWebRtcStream(); 1945} 1946 1947void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRtpExtensions( 1948 const std::vector<webrtc::RtpExtension>& extensions) { 1949 config_.rtp.extensions = extensions; 1950 RecreateWebRtcStream(); 1951} 1952 1953void WebRtcVideoChannel2::WebRtcVideoReceiveStream::RecreateWebRtcStream() { 1954 if (stream_ != NULL) { 1955 call_->DestroyVideoReceiveStream(stream_); 1956 } 1957 stream_ = call_->CreateVideoReceiveStream(config_); 1958 stream_->Start(); 1959} 1960 1961void WebRtcVideoChannel2::WebRtcVideoReceiveStream::ClearDecoders( 1962 std::vector<AllocatedDecoder>* allocated_decoders) { 1963 for (size_t i = 0; i < allocated_decoders->size(); ++i) { 1964 if ((*allocated_decoders)[i].external) { 1965 external_decoder_factory_->DestroyVideoDecoder( 1966 (*allocated_decoders)[i].decoder); 1967 } else { 1968 delete (*allocated_decoders)[i].decoder; 1969 } 1970 } 1971 allocated_decoders->clear(); 1972} 1973 1974void WebRtcVideoChannel2::WebRtcVideoReceiveStream::RenderFrame( 1975 const webrtc::I420VideoFrame& frame, 1976 int time_to_render_ms) { 1977 rtc::CritScope crit(&renderer_lock_); 1978 1979 if (first_frame_timestamp_ < 0) 1980 first_frame_timestamp_ = frame.timestamp(); 1981 int64_t rtp_time_elapsed_since_first_frame = 1982 (timestamp_wraparound_handler_.Unwrap(frame.timestamp()) - 1983 first_frame_timestamp_); 1984 int64_t elapsed_time_ms = rtp_time_elapsed_since_first_frame / 1985 (cricket::kVideoCodecClockrate / 1000); 1986 if (frame.ntp_time_ms() > 0) 1987 estimated_remote_start_ntp_time_ms_ = frame.ntp_time_ms() - elapsed_time_ms; 1988 1989 if (renderer_ == NULL) { 1990 LOG(LS_WARNING) << "VideoReceiveStream not connected to a VideoRenderer."; 1991 return; 1992 } 1993 1994 if (frame.width() != last_width_ || frame.height() != last_height_) { 1995 SetSize(frame.width(), frame.height()); 1996 } 1997 1998 LOG(LS_VERBOSE) << "RenderFrame: (" << frame.width() << "x" << frame.height() 1999 << ")"; 2000 2001 const WebRtcVideoRenderFrame render_frame(&frame, elapsed_time_ms); 2002 renderer_->RenderFrame(&render_frame); 2003} 2004 2005bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::IsTextureSupported() const { 2006 return true; 2007} 2008 2009void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRenderer( 2010 cricket::VideoRenderer* renderer) { 2011 rtc::CritScope crit(&renderer_lock_); 2012 renderer_ = renderer; 2013 if (renderer_ != NULL && last_width_ != -1) { 2014 SetSize(last_width_, last_height_); 2015 } 2016} 2017 2018VideoRenderer* WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetRenderer() { 2019 // TODO(pbos): Remove GetRenderer and all uses of it, it's thread-unsafe by 2020 // design. 2021 rtc::CritScope crit(&renderer_lock_); 2022 return renderer_; 2023} 2024 2025void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetSize(int width, 2026 int height) { 2027 rtc::CritScope crit(&renderer_lock_); 2028 if (!renderer_->SetSize(width, height, 0)) { 2029 LOG(LS_ERROR) << "Could not set renderer size."; 2030 } 2031 last_width_ = width; 2032 last_height_ = height; 2033} 2034 2035VideoReceiverInfo 2036WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetVideoReceiverInfo() { 2037 VideoReceiverInfo info; 2038 info.add_ssrc(config_.rtp.remote_ssrc); 2039 webrtc::VideoReceiveStream::Stats stats = stream_->GetStats(); 2040 info.bytes_rcvd = stats.rtp_stats.transmitted.payload_bytes + 2041 stats.rtp_stats.transmitted.header_bytes + 2042 stats.rtp_stats.transmitted.padding_bytes; 2043 info.packets_rcvd = stats.rtp_stats.transmitted.packets; 2044 2045 info.framerate_rcvd = stats.network_frame_rate; 2046 info.framerate_decoded = stats.decode_frame_rate; 2047 info.framerate_output = stats.render_frame_rate; 2048 2049 { 2050 rtc::CritScope frame_cs(&renderer_lock_); 2051 info.frame_width = last_width_; 2052 info.frame_height = last_height_; 2053 info.capture_start_ntp_time_ms = estimated_remote_start_ntp_time_ms_; 2054 } 2055 2056 info.decode_ms = stats.decode_ms; 2057 info.max_decode_ms = stats.max_decode_ms; 2058 info.current_delay_ms = stats.current_delay_ms; 2059 info.target_delay_ms = stats.target_delay_ms; 2060 info.jitter_buffer_ms = stats.jitter_buffer_ms; 2061 info.min_playout_delay_ms = stats.min_playout_delay_ms; 2062 info.render_delay_ms = stats.render_delay_ms; 2063 2064 info.firs_sent = stats.rtcp_packet_type_counts.fir_packets; 2065 info.plis_sent = stats.rtcp_packet_type_counts.pli_packets; 2066 info.nacks_sent = stats.rtcp_packet_type_counts.nack_packets; 2067 2068 return info; 2069} 2070 2071WebRtcVideoChannel2::VideoCodecSettings::VideoCodecSettings() 2072 : rtx_payload_type(-1) {} 2073 2074bool WebRtcVideoChannel2::VideoCodecSettings::operator==( 2075 const WebRtcVideoChannel2::VideoCodecSettings& other) const { 2076 return codec == other.codec && 2077 fec.ulpfec_payload_type == other.fec.ulpfec_payload_type && 2078 fec.red_payload_type == other.fec.red_payload_type && 2079 rtx_payload_type == other.rtx_payload_type; 2080} 2081 2082std::vector<WebRtcVideoChannel2::VideoCodecSettings> 2083WebRtcVideoChannel2::MapCodecs(const std::vector<VideoCodec>& codecs) { 2084 assert(!codecs.empty()); 2085 2086 std::vector<VideoCodecSettings> video_codecs; 2087 std::map<int, bool> payload_used; 2088 std::map<int, VideoCodec::CodecType> payload_codec_type; 2089 // |rtx_mapping| maps video payload type to rtx payload type. 2090 std::map<int, int> rtx_mapping; 2091 2092 webrtc::FecConfig fec_settings; 2093 2094 for (size_t i = 0; i < codecs.size(); ++i) { 2095 const VideoCodec& in_codec = codecs[i]; 2096 int payload_type = in_codec.id; 2097 2098 if (payload_used[payload_type]) { 2099 LOG(LS_ERROR) << "Payload type already registered: " 2100 << in_codec.ToString(); 2101 return std::vector<VideoCodecSettings>(); 2102 } 2103 payload_used[payload_type] = true; 2104 payload_codec_type[payload_type] = in_codec.GetCodecType(); 2105 2106 switch (in_codec.GetCodecType()) { 2107 case VideoCodec::CODEC_RED: { 2108 // RED payload type, should not have duplicates. 2109 assert(fec_settings.red_payload_type == -1); 2110 fec_settings.red_payload_type = in_codec.id; 2111 continue; 2112 } 2113 2114 case VideoCodec::CODEC_ULPFEC: { 2115 // ULPFEC payload type, should not have duplicates. 2116 assert(fec_settings.ulpfec_payload_type == -1); 2117 fec_settings.ulpfec_payload_type = in_codec.id; 2118 continue; 2119 } 2120 2121 case VideoCodec::CODEC_RTX: { 2122 int associated_payload_type; 2123 if (!in_codec.GetParam(kCodecParamAssociatedPayloadType, 2124 &associated_payload_type) || 2125 !IsValidRtpPayloadType(associated_payload_type)) { 2126 LOG(LS_ERROR) 2127 << "RTX codec with invalid or no associated payload type: " 2128 << in_codec.ToString(); 2129 return std::vector<VideoCodecSettings>(); 2130 } 2131 rtx_mapping[associated_payload_type] = in_codec.id; 2132 continue; 2133 } 2134 2135 case VideoCodec::CODEC_VIDEO: 2136 break; 2137 } 2138 2139 video_codecs.push_back(VideoCodecSettings()); 2140 video_codecs.back().codec = in_codec; 2141 } 2142 2143 // One of these codecs should have been a video codec. Only having FEC 2144 // parameters into this code is a logic error. 2145 assert(!video_codecs.empty()); 2146 2147 for (std::map<int, int>::const_iterator it = rtx_mapping.begin(); 2148 it != rtx_mapping.end(); 2149 ++it) { 2150 if (!payload_used[it->first]) { 2151 LOG(LS_ERROR) << "RTX mapped to payload not in codec list."; 2152 return std::vector<VideoCodecSettings>(); 2153 } 2154 if (payload_codec_type[it->first] != VideoCodec::CODEC_VIDEO) { 2155 LOG(LS_ERROR) << "RTX not mapped to regular video codec."; 2156 return std::vector<VideoCodecSettings>(); 2157 } 2158 } 2159 2160 // TODO(pbos): Write tests that figure out that I have not verified that RTX 2161 // codecs aren't mapped to bogus payloads. 2162 for (size_t i = 0; i < video_codecs.size(); ++i) { 2163 video_codecs[i].fec = fec_settings; 2164 if (rtx_mapping[video_codecs[i].codec.id] != 0) { 2165 video_codecs[i].rtx_payload_type = rtx_mapping[video_codecs[i].codec.id]; 2166 } 2167 } 2168 2169 return video_codecs; 2170} 2171 2172} // namespace cricket 2173 2174#endif // HAVE_WEBRTC_VIDEO 2175