webrtcvideoengine2.cc revision 09c77b95bb62566be64da662f0b3b6a838ec6553
1/* 2 * libjingle 3 * Copyright 2014 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#ifdef HAVE_WEBRTC_VIDEO 29#include "talk/media/webrtc/webrtcvideoengine2.h" 30 31#include <algorithm> 32#include <set> 33#include <string> 34 35#include "libyuv/convert_from.h" 36#include "talk/media/base/videocapturer.h" 37#include "talk/media/base/videorenderer.h" 38#include "talk/media/webrtc/constants.h" 39#include "talk/media/webrtc/simulcast.h" 40#include "talk/media/webrtc/webrtcvideocapturer.h" 41#include "talk/media/webrtc/webrtcvideoengine.h" 42#include "talk/media/webrtc/webrtcvideoframe.h" 43#include "talk/media/webrtc/webrtcvoiceengine.h" 44#include "webrtc/base/buffer.h" 45#include "webrtc/base/logging.h" 46#include "webrtc/base/stringutils.h" 47#include "webrtc/call.h" 48#include "webrtc/system_wrappers/interface/trace_event.h" 49#include "webrtc/video_decoder.h" 50#include "webrtc/video_encoder.h" 51 52#define UNIMPLEMENTED \ 53 LOG(LS_ERROR) << "Call to unimplemented function " << __FUNCTION__; \ 54 ASSERT(false) 55 56namespace cricket { 57namespace { 58static std::string CodecVectorToString(const std::vector<VideoCodec>& codecs) { 59 std::stringstream out; 60 out << '{'; 61 for (size_t i = 0; i < codecs.size(); ++i) { 62 out << codecs[i].ToString(); 63 if (i != codecs.size() - 1) { 64 out << ", "; 65 } 66 } 67 out << '}'; 68 return out.str(); 69} 70 71static bool ValidateCodecFormats(const std::vector<VideoCodec>& codecs) { 72 bool has_video = false; 73 for (size_t i = 0; i < codecs.size(); ++i) { 74 if (!codecs[i].ValidateCodecFormat()) { 75 return false; 76 } 77 if (codecs[i].GetCodecType() == VideoCodec::CODEC_VIDEO) { 78 has_video = true; 79 } 80 } 81 if (!has_video) { 82 LOG(LS_ERROR) << "Setting codecs without a video codec is invalid: " 83 << CodecVectorToString(codecs); 84 return false; 85 } 86 return true; 87} 88 89static std::string RtpExtensionsToString( 90 const std::vector<RtpHeaderExtension>& extensions) { 91 std::stringstream out; 92 out << '{'; 93 for (size_t i = 0; i < extensions.size(); ++i) { 94 out << "{" << extensions[i].uri << ": " << extensions[i].id << "}"; 95 if (i != extensions.size() - 1) { 96 out << ", "; 97 } 98 } 99 out << '}'; 100 return out.str(); 101} 102 103// Merges two fec configs and logs an error if a conflict arises 104// such that merging in diferent order would trigger a diferent output. 105static void MergeFecConfig(const webrtc::FecConfig& other, 106 webrtc::FecConfig* output) { 107 if (other.ulpfec_payload_type != -1) { 108 if (output->ulpfec_payload_type != -1 && 109 output->ulpfec_payload_type != other.ulpfec_payload_type) { 110 LOG(LS_WARNING) << "Conflict merging ulpfec_payload_type configs: " 111 << output->ulpfec_payload_type << " and " 112 << other.ulpfec_payload_type; 113 } 114 output->ulpfec_payload_type = other.ulpfec_payload_type; 115 } 116 if (other.red_payload_type != -1) { 117 if (output->red_payload_type != -1 && 118 output->red_payload_type != other.red_payload_type) { 119 LOG(LS_WARNING) << "Conflict merging red_payload_type configs: " 120 << output->red_payload_type << " and " 121 << other.red_payload_type; 122 } 123 output->red_payload_type = other.red_payload_type; 124 } 125} 126} // namespace 127 128// This constant is really an on/off, lower-level configurable NACK history 129// duration hasn't been implemented. 130static const int kNackHistoryMs = 1000; 131 132static const int kDefaultQpMax = 56; 133 134static const int kDefaultRtcpReceiverReportSsrc = 1; 135 136const char kH264CodecName[] = "H264"; 137 138static bool FindFirstMatchingCodec(const std::vector<VideoCodec>& codecs, 139 const VideoCodec& requested_codec, 140 VideoCodec* matching_codec) { 141 for (size_t i = 0; i < codecs.size(); ++i) { 142 if (requested_codec.Matches(codecs[i])) { 143 *matching_codec = codecs[i]; 144 return true; 145 } 146 } 147 return false; 148} 149 150static bool ValidateRtpHeaderExtensionIds( 151 const std::vector<RtpHeaderExtension>& extensions) { 152 std::set<int> extensions_used; 153 for (size_t i = 0; i < extensions.size(); ++i) { 154 if (extensions[i].id < 0 || extensions[i].id >= 15 || 155 !extensions_used.insert(extensions[i].id).second) { 156 LOG(LS_ERROR) << "RTP extensions are with incorrect or duplicate ids."; 157 return false; 158 } 159 } 160 return true; 161} 162 163static bool CompareRtpHeaderExtensionIds( 164 const webrtc::RtpExtension& extension1, 165 const webrtc::RtpExtension& extension2) { 166 // Sorting on ID is sufficient, more than one extension per ID is unsupported. 167 return extension1.id > extension2.id; 168} 169 170static std::vector<webrtc::RtpExtension> FilterRtpExtensions( 171 const std::vector<RtpHeaderExtension>& extensions) { 172 std::vector<webrtc::RtpExtension> webrtc_extensions; 173 for (size_t i = 0; i < extensions.size(); ++i) { 174 // Unsupported extensions will be ignored. 175 if (webrtc::RtpExtension::IsSupported(extensions[i].uri)) { 176 webrtc_extensions.push_back(webrtc::RtpExtension( 177 extensions[i].uri, extensions[i].id)); 178 } else { 179 LOG(LS_WARNING) << "Unsupported RTP extension: " << extensions[i].uri; 180 } 181 } 182 183 // Sort filtered headers to make sure that they can later be compared 184 // regardless of in which order they were entered. 185 std::sort(webrtc_extensions.begin(), webrtc_extensions.end(), 186 CompareRtpHeaderExtensionIds); 187 return webrtc_extensions; 188} 189 190static bool RtpExtensionsHaveChanged( 191 const std::vector<webrtc::RtpExtension>& before, 192 const std::vector<webrtc::RtpExtension>& after) { 193 if (before.size() != after.size()) 194 return true; 195 for (size_t i = 0; i < before.size(); ++i) { 196 if (before[i].id != after[i].id) 197 return true; 198 if (before[i].name != after[i].name) 199 return true; 200 } 201 return false; 202} 203 204std::vector<webrtc::VideoStream> 205WebRtcVideoChannel2::WebRtcVideoSendStream::CreateSimulcastVideoStreams( 206 const VideoCodec& codec, 207 const VideoOptions& options, 208 size_t num_streams) { 209 // Use default factory for non-simulcast. 210 int max_qp = kDefaultQpMax; 211 codec.GetParam(kCodecParamMaxQuantization, &max_qp); 212 213 int min_bitrate_kbps; 214 if (!codec.GetParam(kCodecParamMinBitrate, &min_bitrate_kbps) || 215 min_bitrate_kbps < kMinVideoBitrate) { 216 min_bitrate_kbps = kMinVideoBitrate; 217 } 218 219 int max_bitrate_kbps; 220 if (!codec.GetParam(kCodecParamMaxBitrate, &max_bitrate_kbps)) { 221 max_bitrate_kbps = 0; 222 } 223 224 return GetSimulcastConfig( 225 num_streams, 226 GetSimulcastBitrateMode(options), 227 codec.width, 228 codec.height, 229 max_bitrate_kbps * 1000, 230 max_qp, 231 codec.framerate != 0 ? codec.framerate : kDefaultVideoMaxFramerate); 232} 233 234std::vector<webrtc::VideoStream> 235WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoStreams( 236 const VideoCodec& codec, 237 const VideoOptions& options, 238 size_t num_streams) { 239 if (num_streams != 1) 240 return CreateSimulcastVideoStreams(codec, options, num_streams); 241 242 webrtc::VideoStream stream; 243 stream.width = codec.width; 244 stream.height = codec.height; 245 stream.max_framerate = 246 codec.framerate != 0 ? codec.framerate : kDefaultVideoMaxFramerate; 247 248 stream.min_bitrate_bps = kMinVideoBitrate * 1000; 249 stream.target_bitrate_bps = stream.max_bitrate_bps = kMaxVideoBitrate * 1000; 250 251 int max_qp = kDefaultQpMax; 252 codec.GetParam(kCodecParamMaxQuantization, &max_qp); 253 stream.max_qp = max_qp; 254 std::vector<webrtc::VideoStream> streams; 255 streams.push_back(stream); 256 return streams; 257} 258 259void* WebRtcVideoChannel2::WebRtcVideoSendStream::ConfigureVideoEncoderSettings( 260 const VideoCodec& codec, 261 const VideoOptions& options) { 262 if (CodecNameMatches(codec.name, kVp8CodecName)) { 263 encoder_settings_.vp8 = webrtc::VideoEncoder::GetDefaultVp8Settings(); 264 options.video_noise_reduction.Get(&encoder_settings_.vp8.denoisingOn); 265 return &encoder_settings_.vp8; 266 } 267 if (CodecNameMatches(codec.name, kVp9CodecName)) { 268 encoder_settings_.vp9 = webrtc::VideoEncoder::GetDefaultVp9Settings(); 269 options.video_noise_reduction.Get(&encoder_settings_.vp9.denoisingOn); 270 return &encoder_settings_.vp9; 271 } 272 return NULL; 273} 274 275DefaultUnsignalledSsrcHandler::DefaultUnsignalledSsrcHandler() 276 : default_recv_ssrc_(0), default_renderer_(NULL) {} 277 278UnsignalledSsrcHandler::Action DefaultUnsignalledSsrcHandler::OnUnsignalledSsrc( 279 VideoMediaChannel* channel, 280 uint32_t ssrc) { 281 if (default_recv_ssrc_ != 0) { // Already one default stream. 282 LOG(LS_WARNING) << "Unknown SSRC, but default receive stream already set."; 283 return kDropPacket; 284 } 285 286 StreamParams sp; 287 sp.ssrcs.push_back(ssrc); 288 LOG(LS_INFO) << "Creating default receive stream for SSRC=" << ssrc << "."; 289 if (!channel->AddRecvStream(sp)) { 290 LOG(LS_WARNING) << "Could not create default receive stream."; 291 } 292 293 channel->SetRenderer(ssrc, default_renderer_); 294 default_recv_ssrc_ = ssrc; 295 return kDeliverPacket; 296} 297 298WebRtcCallFactory::~WebRtcCallFactory() { 299} 300webrtc::Call* WebRtcCallFactory::CreateCall( 301 const webrtc::Call::Config& config) { 302 return webrtc::Call::Create(config); 303} 304 305VideoRenderer* DefaultUnsignalledSsrcHandler::GetDefaultRenderer() const { 306 return default_renderer_; 307} 308 309void DefaultUnsignalledSsrcHandler::SetDefaultRenderer( 310 VideoMediaChannel* channel, 311 VideoRenderer* renderer) { 312 default_renderer_ = renderer; 313 if (default_recv_ssrc_ != 0) { 314 channel->SetRenderer(default_recv_ssrc_, default_renderer_); 315 } 316} 317 318WebRtcVideoEngine2::WebRtcVideoEngine2() 319 : worker_thread_(NULL), 320 voice_engine_(NULL), 321 default_codec_format_(kDefaultVideoMaxWidth, 322 kDefaultVideoMaxHeight, 323 FPS_TO_INTERVAL(kDefaultVideoMaxFramerate), 324 FOURCC_ANY), 325 initialized_(false), 326 call_factory_(&default_call_factory_), 327 external_decoder_factory_(NULL), 328 external_encoder_factory_(NULL) { 329 LOG(LS_INFO) << "WebRtcVideoEngine2::WebRtcVideoEngine2()"; 330 video_codecs_ = GetSupportedCodecs(); 331 rtp_header_extensions_.push_back( 332 RtpHeaderExtension(kRtpTimestampOffsetHeaderExtension, 333 kRtpTimestampOffsetHeaderExtensionDefaultId)); 334 rtp_header_extensions_.push_back( 335 RtpHeaderExtension(kRtpAbsoluteSenderTimeHeaderExtension, 336 kRtpAbsoluteSenderTimeHeaderExtensionDefaultId)); 337} 338 339WebRtcVideoEngine2::~WebRtcVideoEngine2() { 340 LOG(LS_INFO) << "WebRtcVideoEngine2::~WebRtcVideoEngine2"; 341 342 if (initialized_) { 343 Terminate(); 344 } 345} 346 347void WebRtcVideoEngine2::SetCallFactory(WebRtcCallFactory* call_factory) { 348 assert(!initialized_); 349 call_factory_ = call_factory; 350} 351 352bool WebRtcVideoEngine2::Init(rtc::Thread* worker_thread) { 353 LOG(LS_INFO) << "WebRtcVideoEngine2::Init"; 354 worker_thread_ = worker_thread; 355 ASSERT(worker_thread_ != NULL); 356 357 initialized_ = true; 358 return true; 359} 360 361void WebRtcVideoEngine2::Terminate() { 362 LOG(LS_INFO) << "WebRtcVideoEngine2::Terminate"; 363 364 initialized_ = false; 365} 366 367int WebRtcVideoEngine2::GetCapabilities() { return VIDEO_RECV | VIDEO_SEND; } 368 369bool WebRtcVideoEngine2::SetDefaultEncoderConfig( 370 const VideoEncoderConfig& config) { 371 const VideoCodec& codec = config.max_codec; 372 bool supports_codec = false; 373 for (size_t i = 0; i < video_codecs_.size(); ++i) { 374 if (CodecNameMatches(video_codecs_[i].name, codec.name)) { 375 video_codecs_[i] = codec; 376 supports_codec = true; 377 break; 378 } 379 } 380 381 if (!supports_codec) { 382 LOG(LS_ERROR) << "SetDefaultEncoderConfig, codec not supported: " 383 << codec.ToString(); 384 return false; 385 } 386 387 default_codec_format_ = 388 VideoFormat(codec.width, 389 codec.height, 390 VideoFormat::FpsToInterval(codec.framerate), 391 FOURCC_ANY); 392 return true; 393} 394 395WebRtcVideoChannel2* WebRtcVideoEngine2::CreateChannel( 396 const VideoOptions& options, 397 VoiceMediaChannel* voice_channel) { 398 assert(initialized_); 399 LOG(LS_INFO) << "CreateChannel: " 400 << (voice_channel != NULL ? "With" : "Without") 401 << " voice channel. Options: " << options.ToString(); 402 WebRtcVideoChannel2* channel = 403 new WebRtcVideoChannel2(call_factory_, 404 voice_engine_, 405 voice_channel, 406 options, 407 external_encoder_factory_, 408 external_decoder_factory_); 409 if (!channel->Init()) { 410 delete channel; 411 return NULL; 412 } 413 channel->SetRecvCodecs(video_codecs_); 414 return channel; 415} 416 417const std::vector<VideoCodec>& WebRtcVideoEngine2::codecs() const { 418 return video_codecs_; 419} 420 421const std::vector<RtpHeaderExtension>& 422WebRtcVideoEngine2::rtp_header_extensions() const { 423 return rtp_header_extensions_; 424} 425 426void WebRtcVideoEngine2::SetLogging(int min_sev, const char* filter) { 427 // TODO(pbos): Set up logging. 428 LOG(LS_VERBOSE) << "SetLogging: " << min_sev << '"' << filter << '"'; 429 // if min_sev == -1, we keep the current log level. 430 if (min_sev < 0) { 431 assert(min_sev == -1); 432 return; 433 } 434} 435 436void WebRtcVideoEngine2::SetExternalDecoderFactory( 437 WebRtcVideoDecoderFactory* decoder_factory) { 438 assert(!initialized_); 439 external_decoder_factory_ = decoder_factory; 440} 441 442void WebRtcVideoEngine2::SetExternalEncoderFactory( 443 WebRtcVideoEncoderFactory* encoder_factory) { 444 assert(!initialized_); 445 if (external_encoder_factory_ == encoder_factory) 446 return; 447 448 // No matter what happens we shouldn't hold on to a stale 449 // WebRtcSimulcastEncoderFactory. 450 simulcast_encoder_factory_.reset(); 451 452 if (encoder_factory && 453 WebRtcSimulcastEncoderFactory::UseSimulcastEncoderFactory( 454 encoder_factory->codecs())) { 455 simulcast_encoder_factory_.reset( 456 new WebRtcSimulcastEncoderFactory(encoder_factory)); 457 encoder_factory = simulcast_encoder_factory_.get(); 458 } 459 external_encoder_factory_ = encoder_factory; 460 461 video_codecs_ = GetSupportedCodecs(); 462} 463 464bool WebRtcVideoEngine2::EnableTimedRender() { 465 // TODO(pbos): Figure out whether this can be removed. 466 return true; 467} 468 469// Checks to see whether we comprehend and could receive a particular codec 470bool WebRtcVideoEngine2::FindCodec(const VideoCodec& in) { 471 // TODO(pbos): Probe encoder factory to figure out that the codec is supported 472 // if supported by the encoder factory. Add a corresponding test that fails 473 // with this code (that doesn't ask the factory). 474 for (size_t j = 0; j < video_codecs_.size(); ++j) { 475 VideoCodec codec(video_codecs_[j].id, video_codecs_[j].name, 0, 0, 0, 0); 476 if (codec.Matches(in)) { 477 return true; 478 } 479 } 480 return false; 481} 482 483// Tells whether the |requested| codec can be transmitted or not. If it can be 484// transmitted |out| is set with the best settings supported. Aspect ratio will 485// be set as close to |current|'s as possible. If not set |requested|'s 486// dimensions will be used for aspect ratio matching. 487bool WebRtcVideoEngine2::CanSendCodec(const VideoCodec& requested, 488 const VideoCodec& current, 489 VideoCodec* out) { 490 assert(out != NULL); 491 492 if (requested.width != requested.height && 493 (requested.height == 0 || requested.width == 0)) { 494 // 0xn and nx0 are invalid resolutions. 495 return false; 496 } 497 498 VideoCodec matching_codec; 499 if (!FindFirstMatchingCodec(video_codecs_, requested, &matching_codec)) { 500 // Codec not supported. 501 return false; 502 } 503 504 out->id = requested.id; 505 out->name = requested.name; 506 out->preference = requested.preference; 507 out->params = requested.params; 508 out->framerate = std::min(requested.framerate, matching_codec.framerate); 509 out->params = requested.params; 510 out->feedback_params = requested.feedback_params; 511 out->width = requested.width; 512 out->height = requested.height; 513 if (requested.width == 0 && requested.height == 0) { 514 return true; 515 } 516 517 while (out->width > matching_codec.width) { 518 out->width /= 2; 519 out->height /= 2; 520 } 521 522 return out->width > 0 && out->height > 0; 523} 524 525bool WebRtcVideoEngine2::SetVoiceEngine(WebRtcVoiceEngine* voice_engine) { 526 if (initialized_) { 527 LOG(LS_WARNING) << "SetVoiceEngine can not be called after Init"; 528 return false; 529 } 530 voice_engine_ = voice_engine; 531 return true; 532} 533 534// Ignore spammy trace messages, mostly from the stats API when we haven't 535// gotten RTCP info yet from the remote side. 536bool WebRtcVideoEngine2::ShouldIgnoreTrace(const std::string& trace) { 537 static const char* const kTracesToIgnore[] = {NULL}; 538 for (const char* const* p = kTracesToIgnore; *p; ++p) { 539 if (trace.find(*p) == 0) { 540 return true; 541 } 542 } 543 return false; 544} 545 546std::vector<VideoCodec> WebRtcVideoEngine2::GetSupportedCodecs() const { 547 std::vector<VideoCodec> supported_codecs = DefaultVideoCodecList(); 548 549 if (external_encoder_factory_ == NULL) { 550 return supported_codecs; 551 } 552 553 const std::vector<WebRtcVideoEncoderFactory::VideoCodec>& codecs = 554 external_encoder_factory_->codecs(); 555 for (size_t i = 0; i < codecs.size(); ++i) { 556 // Don't add internally-supported codecs twice. 557 if (CodecIsInternallySupported(codecs[i].name)) { 558 continue; 559 } 560 561 // External video encoders are given payloads 120-127. This also means that 562 // we only support up to 8 external payload types. 563 const int kExternalVideoPayloadTypeBase = 120; 564 size_t payload_type = kExternalVideoPayloadTypeBase + i; 565 assert(payload_type < 128); 566 VideoCodec codec(static_cast<int>(payload_type), 567 codecs[i].name, 568 codecs[i].max_width, 569 codecs[i].max_height, 570 codecs[i].max_fps, 571 0); 572 573 AddDefaultFeedbackParams(&codec); 574 supported_codecs.push_back(codec); 575 } 576 return supported_codecs; 577} 578 579WebRtcVideoChannel2::WebRtcVideoChannel2( 580 WebRtcCallFactory* call_factory, 581 WebRtcVoiceEngine* voice_engine, 582 VoiceMediaChannel* voice_channel, 583 const VideoOptions& options, 584 WebRtcVideoEncoderFactory* external_encoder_factory, 585 WebRtcVideoDecoderFactory* external_decoder_factory) 586 : unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_), 587 voice_channel_(voice_channel), 588 external_encoder_factory_(external_encoder_factory), 589 external_decoder_factory_(external_decoder_factory) { 590 SetDefaultOptions(); 591 options_.SetAll(options); 592 webrtc::Call::Config config(this); 593 config.overuse_callback = this; 594 if (voice_engine != NULL) { 595 config.voice_engine = voice_engine->voe()->engine(); 596 } 597 598 call_.reset(call_factory->CreateCall(config)); 599 600 rtcp_receiver_report_ssrc_ = kDefaultRtcpReceiverReportSsrc; 601 sending_ = false; 602 default_send_ssrc_ = 0; 603} 604 605void WebRtcVideoChannel2::SetDefaultOptions() { 606 options_.cpu_overuse_detection.Set(false); 607 options_.dscp.Set(false); 608 options_.suspend_below_min_bitrate.Set(false); 609 options_.video_noise_reduction.Set(true); 610 options_.screencast_min_bitrate.Set(0); 611} 612 613WebRtcVideoChannel2::~WebRtcVideoChannel2() { 614 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 615 send_streams_.begin(); 616 it != send_streams_.end(); 617 ++it) { 618 delete it->second; 619 } 620 621 for (std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 622 receive_streams_.begin(); 623 it != receive_streams_.end(); 624 ++it) { 625 delete it->second; 626 } 627} 628 629bool WebRtcVideoChannel2::Init() { return true; } 630 631bool WebRtcVideoChannel2::CodecIsExternallySupported( 632 const std::string& name) const { 633 if (external_encoder_factory_ == NULL) { 634 return false; 635 } 636 637 const std::vector<WebRtcVideoEncoderFactory::VideoCodec> external_codecs = 638 external_encoder_factory_->codecs(); 639 for (size_t c = 0; c < external_codecs.size(); ++c) { 640 if (CodecNameMatches(name, external_codecs[c].name)) { 641 return true; 642 } 643 } 644 return false; 645} 646 647std::vector<WebRtcVideoChannel2::VideoCodecSettings> 648WebRtcVideoChannel2::FilterSupportedCodecs( 649 const std::vector<WebRtcVideoChannel2::VideoCodecSettings>& mapped_codecs) 650 const { 651 std::vector<VideoCodecSettings> supported_codecs; 652 for (size_t i = 0; i < mapped_codecs.size(); ++i) { 653 const VideoCodecSettings& codec = mapped_codecs[i]; 654 if (CodecIsInternallySupported(codec.codec.name) || 655 CodecIsExternallySupported(codec.codec.name)) { 656 supported_codecs.push_back(codec); 657 } 658 } 659 return supported_codecs; 660} 661 662bool WebRtcVideoChannel2::SetRecvCodecs(const std::vector<VideoCodec>& codecs) { 663 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetRecvCodecs"); 664 LOG(LS_INFO) << "SetRecvCodecs: " << CodecVectorToString(codecs); 665 if (!ValidateCodecFormats(codecs)) { 666 return false; 667 } 668 669 const std::vector<VideoCodecSettings> mapped_codecs = MapCodecs(codecs); 670 if (mapped_codecs.empty()) { 671 LOG(LS_ERROR) << "SetRecvCodecs called without any video codecs."; 672 return false; 673 } 674 675 const std::vector<VideoCodecSettings> supported_codecs = 676 FilterSupportedCodecs(mapped_codecs); 677 678 if (mapped_codecs.size() != supported_codecs.size()) { 679 LOG(LS_ERROR) << "SetRecvCodecs called with unsupported video codecs."; 680 return false; 681 } 682 683 recv_codecs_ = supported_codecs; 684 685 rtc::CritScope stream_lock(&stream_crit_); 686 for (std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 687 receive_streams_.begin(); 688 it != receive_streams_.end(); 689 ++it) { 690 it->second->SetRecvCodecs(recv_codecs_); 691 } 692 693 return true; 694} 695 696bool WebRtcVideoChannel2::SetSendCodecs(const std::vector<VideoCodec>& codecs) { 697 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetSendCodecs"); 698 LOG(LS_INFO) << "SetSendCodecs: " << CodecVectorToString(codecs); 699 if (!ValidateCodecFormats(codecs)) { 700 return false; 701 } 702 703 const std::vector<VideoCodecSettings> supported_codecs = 704 FilterSupportedCodecs(MapCodecs(codecs)); 705 706 if (supported_codecs.empty()) { 707 LOG(LS_ERROR) << "No video codecs supported by encoder factory."; 708 return false; 709 } 710 711 LOG(LS_INFO) << "Using codec: " << supported_codecs.front().codec.ToString(); 712 713 VideoCodecSettings old_codec; 714 if (send_codec_.Get(&old_codec) && supported_codecs.front() == old_codec) { 715 // Using same codec, avoid reconfiguring. 716 return true; 717 } 718 719 send_codec_.Set(supported_codecs.front()); 720 721 rtc::CritScope stream_lock(&stream_crit_); 722 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 723 send_streams_.begin(); 724 it != send_streams_.end(); 725 ++it) { 726 assert(it->second != NULL); 727 it->second->SetCodec(supported_codecs.front()); 728 } 729 730 VideoCodec codec = supported_codecs.front().codec; 731 int bitrate_kbps; 732 if (codec.GetParam(kCodecParamMinBitrate, &bitrate_kbps) && 733 bitrate_kbps > 0) { 734 bitrate_config_.min_bitrate_bps = bitrate_kbps * 1000; 735 } else { 736 bitrate_config_.min_bitrate_bps = 0; 737 } 738 if (codec.GetParam(kCodecParamStartBitrate, &bitrate_kbps) && 739 bitrate_kbps > 0) { 740 bitrate_config_.start_bitrate_bps = bitrate_kbps * 1000; 741 } else { 742 // Do not reconfigure start bitrate unless it's specified and positive. 743 bitrate_config_.start_bitrate_bps = -1; 744 } 745 if (codec.GetParam(kCodecParamMaxBitrate, &bitrate_kbps) && 746 bitrate_kbps > 0) { 747 bitrate_config_.max_bitrate_bps = bitrate_kbps * 1000; 748 } else { 749 bitrate_config_.max_bitrate_bps = -1; 750 } 751 call_->SetBitrateConfig(bitrate_config_); 752 753 return true; 754} 755 756bool WebRtcVideoChannel2::GetSendCodec(VideoCodec* codec) { 757 VideoCodecSettings codec_settings; 758 if (!send_codec_.Get(&codec_settings)) { 759 LOG(LS_VERBOSE) << "GetSendCodec: No send codec set."; 760 return false; 761 } 762 *codec = codec_settings.codec; 763 return true; 764} 765 766bool WebRtcVideoChannel2::SetSendStreamFormat(uint32 ssrc, 767 const VideoFormat& format) { 768 LOG(LS_VERBOSE) << "SetSendStreamFormat:" << ssrc << " -> " 769 << format.ToString(); 770 rtc::CritScope stream_lock(&stream_crit_); 771 if (send_streams_.find(ssrc) == send_streams_.end()) { 772 return false; 773 } 774 return send_streams_[ssrc]->SetVideoFormat(format); 775} 776 777bool WebRtcVideoChannel2::SetRender(bool render) { 778 // TODO(pbos): Implement. Or refactor away as it shouldn't be needed. 779 LOG(LS_VERBOSE) << "SetRender: " << (render ? "true" : "false"); 780 return true; 781} 782 783bool WebRtcVideoChannel2::SetSend(bool send) { 784 LOG(LS_VERBOSE) << "SetSend: " << (send ? "true" : "false"); 785 if (send && !send_codec_.IsSet()) { 786 LOG(LS_ERROR) << "SetSend(true) called before setting codec."; 787 return false; 788 } 789 if (send) { 790 StartAllSendStreams(); 791 } else { 792 StopAllSendStreams(); 793 } 794 sending_ = send; 795 return true; 796} 797 798bool WebRtcVideoChannel2::AddSendStream(const StreamParams& sp) { 799 LOG(LS_INFO) << "AddSendStream: " << sp.ToString(); 800 if (sp.ssrcs.empty()) { 801 LOG(LS_ERROR) << "No SSRCs in stream parameters."; 802 return false; 803 } 804 805 uint32 ssrc = sp.first_ssrc(); 806 assert(ssrc != 0); 807 // TODO(pbos): Make sure none of sp.ssrcs are used, not just the identifying 808 // ssrc. 809 rtc::CritScope stream_lock(&stream_crit_); 810 if (send_streams_.find(ssrc) != send_streams_.end()) { 811 LOG(LS_ERROR) << "Send stream with ssrc '" << ssrc << "' already exists."; 812 return false; 813 } 814 815 std::vector<uint32> primary_ssrcs; 816 sp.GetPrimarySsrcs(&primary_ssrcs); 817 std::vector<uint32> rtx_ssrcs; 818 sp.GetFidSsrcs(primary_ssrcs, &rtx_ssrcs); 819 if (!rtx_ssrcs.empty() && primary_ssrcs.size() != rtx_ssrcs.size()) { 820 LOG(LS_ERROR) 821 << "RTX SSRCs exist, but don't cover all SSRCs (unsupported): " 822 << sp.ToString(); 823 return false; 824 } 825 826 WebRtcVideoSendStream* stream = 827 new WebRtcVideoSendStream(call_.get(), 828 external_encoder_factory_, 829 options_, 830 send_codec_, 831 sp, 832 send_rtp_extensions_); 833 834 send_streams_[ssrc] = stream; 835 836 if (rtcp_receiver_report_ssrc_ == kDefaultRtcpReceiverReportSsrc) { 837 rtcp_receiver_report_ssrc_ = ssrc; 838 } 839 if (default_send_ssrc_ == 0) { 840 default_send_ssrc_ = ssrc; 841 } 842 if (sending_) { 843 stream->Start(); 844 } 845 846 return true; 847} 848 849bool WebRtcVideoChannel2::RemoveSendStream(uint32 ssrc) { 850 LOG(LS_INFO) << "RemoveSendStream: " << ssrc; 851 852 if (ssrc == 0) { 853 if (default_send_ssrc_ == 0) { 854 LOG(LS_ERROR) << "No default send stream active."; 855 return false; 856 } 857 858 LOG(LS_VERBOSE) << "Removing default stream: " << default_send_ssrc_; 859 ssrc = default_send_ssrc_; 860 } 861 862 WebRtcVideoSendStream* removed_stream; 863 { 864 rtc::CritScope stream_lock(&stream_crit_); 865 std::map<uint32, WebRtcVideoSendStream*>::iterator it = 866 send_streams_.find(ssrc); 867 if (it == send_streams_.end()) { 868 return false; 869 } 870 871 removed_stream = it->second; 872 send_streams_.erase(it); 873 } 874 875 delete removed_stream; 876 877 if (ssrc == default_send_ssrc_) { 878 default_send_ssrc_ = 0; 879 } 880 881 return true; 882} 883 884bool WebRtcVideoChannel2::AddRecvStream(const StreamParams& sp) { 885 LOG(LS_INFO) << "AddRecvStream: " << sp.ToString(); 886 assert(sp.ssrcs.size() > 0); 887 888 uint32 ssrc = sp.first_ssrc(); 889 assert(ssrc != 0); // TODO(pbos): Is this ever valid? 890 891 // TODO(pbos): Check if any of the SSRCs overlap. 892 rtc::CritScope stream_lock(&stream_crit_); 893 if (receive_streams_.find(ssrc) != receive_streams_.end()) { 894 LOG(LS_ERROR) << "Receive stream for SSRC " << ssrc << "already exists."; 895 return false; 896 } 897 898 webrtc::VideoReceiveStream::Config config; 899 ConfigureReceiverRtp(&config, sp); 900 901 // Set up A/V sync if there is a VoiceChannel. 902 // TODO(pbos): The A/V is synched by the receiving channel. So we need to know 903 // the SSRC of the remote audio channel in order to sync the correct webrtc 904 // VoiceEngine channel. For now sync the first channel in non-conference to 905 // match existing behavior in WebRtcVideoEngine. 906 if (voice_channel_ != NULL && receive_streams_.empty() && 907 !options_.conference_mode.GetWithDefaultIfUnset(false)) { 908 config.audio_channel_id = 909 static_cast<WebRtcVoiceMediaChannel*>(voice_channel_)->voe_channel(); 910 } 911 912 receive_streams_[ssrc] = new WebRtcVideoReceiveStream( 913 call_.get(), external_decoder_factory_, config, recv_codecs_); 914 915 return true; 916} 917 918void WebRtcVideoChannel2::ConfigureReceiverRtp( 919 webrtc::VideoReceiveStream::Config* config, 920 const StreamParams& sp) const { 921 uint32 ssrc = sp.first_ssrc(); 922 923 config->rtp.remote_ssrc = ssrc; 924 config->rtp.local_ssrc = rtcp_receiver_report_ssrc_; 925 926 config->rtp.extensions = recv_rtp_extensions_; 927 928 // TODO(pbos): This protection is against setting the same local ssrc as 929 // remote which is not permitted by the lower-level API. RTCP requires a 930 // corresponding sender SSRC. Figure out what to do when we don't have 931 // (receive-only) or know a good local SSRC. 932 if (config->rtp.remote_ssrc == config->rtp.local_ssrc) { 933 if (config->rtp.local_ssrc != kDefaultRtcpReceiverReportSsrc) { 934 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc; 935 } else { 936 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc + 1; 937 } 938 } 939 940 for (size_t i = 0; i < recv_codecs_.size(); ++i) { 941 MergeFecConfig(recv_codecs_[i].fec, &config->rtp.fec); 942 } 943 944 for (size_t i = 0; i < recv_codecs_.size(); ++i) { 945 uint32 rtx_ssrc; 946 if (recv_codecs_[i].rtx_payload_type != -1 && 947 sp.GetFidSsrc(ssrc, &rtx_ssrc)) { 948 webrtc::VideoReceiveStream::Config::Rtp::Rtx& rtx = 949 config->rtp.rtx[recv_codecs_[i].codec.id]; 950 rtx.ssrc = rtx_ssrc; 951 rtx.payload_type = recv_codecs_[i].rtx_payload_type; 952 } 953 } 954} 955 956bool WebRtcVideoChannel2::RemoveRecvStream(uint32 ssrc) { 957 LOG(LS_INFO) << "RemoveRecvStream: " << ssrc; 958 if (ssrc == 0) { 959 LOG(LS_ERROR) << "RemoveRecvStream with 0 ssrc is not supported."; 960 return false; 961 } 962 963 rtc::CritScope stream_lock(&stream_crit_); 964 std::map<uint32, WebRtcVideoReceiveStream*>::iterator stream = 965 receive_streams_.find(ssrc); 966 if (stream == receive_streams_.end()) { 967 LOG(LS_ERROR) << "Stream not found for ssrc: " << ssrc; 968 return false; 969 } 970 delete stream->second; 971 receive_streams_.erase(stream); 972 973 return true; 974} 975 976bool WebRtcVideoChannel2::SetRenderer(uint32 ssrc, VideoRenderer* renderer) { 977 LOG(LS_INFO) << "SetRenderer: ssrc:" << ssrc << " " 978 << (renderer ? "(ptr)" : "NULL"); 979 if (ssrc == 0) { 980 default_unsignalled_ssrc_handler_.SetDefaultRenderer(this, renderer); 981 return true; 982 } 983 984 rtc::CritScope stream_lock(&stream_crit_); 985 std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 986 receive_streams_.find(ssrc); 987 if (it == receive_streams_.end()) { 988 return false; 989 } 990 991 it->second->SetRenderer(renderer); 992 return true; 993} 994 995bool WebRtcVideoChannel2::GetRenderer(uint32 ssrc, VideoRenderer** renderer) { 996 if (ssrc == 0) { 997 *renderer = default_unsignalled_ssrc_handler_.GetDefaultRenderer(); 998 return *renderer != NULL; 999 } 1000 1001 rtc::CritScope stream_lock(&stream_crit_); 1002 std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 1003 receive_streams_.find(ssrc); 1004 if (it == receive_streams_.end()) { 1005 return false; 1006 } 1007 *renderer = it->second->GetRenderer(); 1008 return true; 1009} 1010 1011bool WebRtcVideoChannel2::GetStats(const StatsOptions& options, 1012 VideoMediaInfo* info) { 1013 info->Clear(); 1014 FillSenderStats(info); 1015 FillReceiverStats(info); 1016 webrtc::Call::Stats stats = call_->GetStats(); 1017 FillBandwidthEstimationStats(stats, info); 1018 if (stats.rtt_ms != -1) { 1019 for (size_t i = 0; i < info->senders.size(); ++i) { 1020 info->senders[i].rtt_ms = stats.rtt_ms; 1021 } 1022 } 1023 return true; 1024} 1025 1026void WebRtcVideoChannel2::FillSenderStats(VideoMediaInfo* video_media_info) { 1027 rtc::CritScope stream_lock(&stream_crit_); 1028 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1029 send_streams_.begin(); 1030 it != send_streams_.end(); 1031 ++it) { 1032 video_media_info->senders.push_back(it->second->GetVideoSenderInfo()); 1033 } 1034} 1035 1036void WebRtcVideoChannel2::FillReceiverStats(VideoMediaInfo* video_media_info) { 1037 rtc::CritScope stream_lock(&stream_crit_); 1038 for (std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 1039 receive_streams_.begin(); 1040 it != receive_streams_.end(); 1041 ++it) { 1042 video_media_info->receivers.push_back(it->second->GetVideoReceiverInfo()); 1043 } 1044} 1045 1046void WebRtcVideoChannel2::FillBandwidthEstimationStats( 1047 const webrtc::Call::Stats& stats, 1048 VideoMediaInfo* video_media_info) { 1049 BandwidthEstimationInfo bwe_info; 1050 bwe_info.available_send_bandwidth = stats.send_bandwidth_bps; 1051 bwe_info.available_recv_bandwidth = stats.recv_bandwidth_bps; 1052 bwe_info.bucket_delay = stats.pacer_delay_ms; 1053 1054 // Get send stream bitrate stats. 1055 rtc::CritScope stream_lock(&stream_crit_); 1056 for (std::map<uint32, WebRtcVideoSendStream*>::iterator stream = 1057 send_streams_.begin(); 1058 stream != send_streams_.end(); 1059 ++stream) { 1060 stream->second->FillBandwidthEstimationInfo(&bwe_info); 1061 } 1062 video_media_info->bw_estimations.push_back(bwe_info); 1063} 1064 1065bool WebRtcVideoChannel2::SetCapturer(uint32 ssrc, VideoCapturer* capturer) { 1066 LOG(LS_INFO) << "SetCapturer: " << ssrc << " -> " 1067 << (capturer != NULL ? "(capturer)" : "NULL"); 1068 assert(ssrc != 0); 1069 rtc::CritScope stream_lock(&stream_crit_); 1070 if (send_streams_.find(ssrc) == send_streams_.end()) { 1071 LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc; 1072 return false; 1073 } 1074 return send_streams_[ssrc]->SetCapturer(capturer); 1075} 1076 1077bool WebRtcVideoChannel2::SendIntraFrame() { 1078 // TODO(pbos): Implement. 1079 LOG(LS_VERBOSE) << "SendIntraFrame()."; 1080 return true; 1081} 1082 1083bool WebRtcVideoChannel2::RequestIntraFrame() { 1084 // TODO(pbos): Implement. 1085 LOG(LS_VERBOSE) << "SendIntraFrame()."; 1086 return true; 1087} 1088 1089void WebRtcVideoChannel2::OnPacketReceived( 1090 rtc::Buffer* packet, 1091 const rtc::PacketTime& packet_time) { 1092 const webrtc::PacketReceiver::DeliveryStatus delivery_result = 1093 call_->Receiver()->DeliverPacket( 1094 reinterpret_cast<const uint8_t*>(packet->data()), packet->length()); 1095 switch (delivery_result) { 1096 case webrtc::PacketReceiver::DELIVERY_OK: 1097 return; 1098 case webrtc::PacketReceiver::DELIVERY_PACKET_ERROR: 1099 return; 1100 case webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC: 1101 break; 1102 } 1103 1104 uint32 ssrc = 0; 1105 if (!GetRtpSsrc(packet->data(), packet->length(), &ssrc)) { 1106 return; 1107 } 1108 1109 // TODO(pbos): Make sure that the unsignalled SSRC uses the video payload. 1110 // Also figure out whether RTX needs to be handled. 1111 switch (unsignalled_ssrc_handler_->OnUnsignalledSsrc(this, ssrc)) { 1112 case UnsignalledSsrcHandler::kDropPacket: 1113 return; 1114 case UnsignalledSsrcHandler::kDeliverPacket: 1115 break; 1116 } 1117 1118 if (call_->Receiver()->DeliverPacket( 1119 reinterpret_cast<const uint8_t*>(packet->data()), packet->length()) != 1120 webrtc::PacketReceiver::DELIVERY_OK) { 1121 LOG(LS_WARNING) << "Failed to deliver RTP packet on re-delivery."; 1122 return; 1123 } 1124} 1125 1126void WebRtcVideoChannel2::OnRtcpReceived( 1127 rtc::Buffer* packet, 1128 const rtc::PacketTime& packet_time) { 1129 if (call_->Receiver()->DeliverPacket( 1130 reinterpret_cast<const uint8_t*>(packet->data()), packet->length()) != 1131 webrtc::PacketReceiver::DELIVERY_OK) { 1132 LOG(LS_WARNING) << "Failed to deliver RTCP packet."; 1133 } 1134} 1135 1136void WebRtcVideoChannel2::OnReadyToSend(bool ready) { 1137 LOG(LS_VERBOSE) << "OnReadyToSend: " << (ready ? "Ready." : "Not ready."); 1138 call_->SignalNetworkState(ready ? webrtc::Call::kNetworkUp 1139 : webrtc::Call::kNetworkDown); 1140} 1141 1142bool WebRtcVideoChannel2::MuteStream(uint32 ssrc, bool mute) { 1143 LOG(LS_VERBOSE) << "MuteStream: " << ssrc << " -> " 1144 << (mute ? "mute" : "unmute"); 1145 assert(ssrc != 0); 1146 rtc::CritScope stream_lock(&stream_crit_); 1147 if (send_streams_.find(ssrc) == send_streams_.end()) { 1148 LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc; 1149 return false; 1150 } 1151 1152 send_streams_[ssrc]->MuteStream(mute); 1153 return true; 1154} 1155 1156bool WebRtcVideoChannel2::SetRecvRtpHeaderExtensions( 1157 const std::vector<RtpHeaderExtension>& extensions) { 1158 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetRecvRtpHeaderExtensions"); 1159 LOG(LS_INFO) << "SetRecvRtpHeaderExtensions: " 1160 << RtpExtensionsToString(extensions); 1161 if (!ValidateRtpHeaderExtensionIds(extensions)) 1162 return false; 1163 1164 std::vector<webrtc::RtpExtension> filtered_extensions = 1165 FilterRtpExtensions(extensions); 1166 if (!RtpExtensionsHaveChanged(recv_rtp_extensions_, filtered_extensions)) 1167 return true; 1168 1169 recv_rtp_extensions_ = filtered_extensions; 1170 1171 rtc::CritScope stream_lock(&stream_crit_); 1172 for (std::map<uint32, WebRtcVideoReceiveStream*>::iterator it = 1173 receive_streams_.begin(); 1174 it != receive_streams_.end(); 1175 ++it) { 1176 it->second->SetRtpExtensions(recv_rtp_extensions_); 1177 } 1178 return true; 1179} 1180 1181bool WebRtcVideoChannel2::SetSendRtpHeaderExtensions( 1182 const std::vector<RtpHeaderExtension>& extensions) { 1183 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetSendRtpHeaderExtensions"); 1184 LOG(LS_INFO) << "SetSendRtpHeaderExtensions: " 1185 << RtpExtensionsToString(extensions); 1186 if (!ValidateRtpHeaderExtensionIds(extensions)) 1187 return false; 1188 1189 std::vector<webrtc::RtpExtension> filtered_extensions = 1190 FilterRtpExtensions(extensions); 1191 if (!RtpExtensionsHaveChanged(send_rtp_extensions_, filtered_extensions)) 1192 return true; 1193 1194 send_rtp_extensions_ = filtered_extensions; 1195 1196 rtc::CritScope stream_lock(&stream_crit_); 1197 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1198 send_streams_.begin(); 1199 it != send_streams_.end(); 1200 ++it) { 1201 it->second->SetRtpExtensions(send_rtp_extensions_); 1202 } 1203 return true; 1204} 1205 1206bool WebRtcVideoChannel2::SetMaxSendBandwidth(int max_bitrate_bps) { 1207 LOG(LS_INFO) << "SetMaxSendBandwidth: " << max_bitrate_bps << "bps."; 1208 if (max_bitrate_bps <= 0) { 1209 // Unsetting max bitrate. 1210 max_bitrate_bps = -1; 1211 } 1212 bitrate_config_.start_bitrate_bps = -1; 1213 bitrate_config_.max_bitrate_bps = max_bitrate_bps; 1214 if (max_bitrate_bps > 0 && 1215 bitrate_config_.min_bitrate_bps > max_bitrate_bps) { 1216 bitrate_config_.min_bitrate_bps = max_bitrate_bps; 1217 } 1218 call_->SetBitrateConfig(bitrate_config_); 1219 return true; 1220} 1221 1222bool WebRtcVideoChannel2::SetOptions(const VideoOptions& options) { 1223 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetOptions"); 1224 LOG(LS_INFO) << "SetOptions: " << options.ToString(); 1225 VideoOptions old_options = options_; 1226 options_.SetAll(options); 1227 if (options_ == old_options) { 1228 // No new options to set. 1229 return true; 1230 } 1231 rtc::DiffServCodePoint dscp = options_.dscp.GetWithDefaultIfUnset(false) 1232 ? rtc::DSCP_AF41 1233 : rtc::DSCP_DEFAULT; 1234 MediaChannel::SetDscp(dscp); 1235 rtc::CritScope stream_lock(&stream_crit_); 1236 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1237 send_streams_.begin(); 1238 it != send_streams_.end(); 1239 ++it) { 1240 it->second->SetOptions(options_); 1241 } 1242 return true; 1243} 1244 1245void WebRtcVideoChannel2::SetInterface(NetworkInterface* iface) { 1246 MediaChannel::SetInterface(iface); 1247 // Set the RTP recv/send buffer to a bigger size 1248 MediaChannel::SetOption(NetworkInterface::ST_RTP, 1249 rtc::Socket::OPT_RCVBUF, 1250 kVideoRtpBufferSize); 1251 1252 // Speculative change to increase the outbound socket buffer size. 1253 // In b/15152257, we are seeing a significant number of packets discarded 1254 // due to lack of socket buffer space, although it's not yet clear what the 1255 // ideal value should be. 1256 MediaChannel::SetOption(NetworkInterface::ST_RTP, 1257 rtc::Socket::OPT_SNDBUF, 1258 kVideoRtpBufferSize); 1259} 1260 1261void WebRtcVideoChannel2::UpdateAspectRatio(int ratio_w, int ratio_h) { 1262 // TODO(pbos): Implement. 1263} 1264 1265void WebRtcVideoChannel2::OnMessage(rtc::Message* msg) { 1266 // Ignored. 1267} 1268 1269void WebRtcVideoChannel2::OnLoadUpdate(Load load) { 1270 rtc::CritScope stream_lock(&stream_crit_); 1271 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1272 send_streams_.begin(); 1273 it != send_streams_.end(); 1274 ++it) { 1275 it->second->OnCpuResolutionRequest(load == kOveruse 1276 ? CoordinatedVideoAdapter::DOWNGRADE 1277 : CoordinatedVideoAdapter::UPGRADE); 1278 } 1279} 1280 1281bool WebRtcVideoChannel2::SendRtp(const uint8_t* data, size_t len) { 1282 rtc::Buffer packet(data, len, kMaxRtpPacketLen); 1283 return MediaChannel::SendPacket(&packet); 1284} 1285 1286bool WebRtcVideoChannel2::SendRtcp(const uint8_t* data, size_t len) { 1287 rtc::Buffer packet(data, len, kMaxRtpPacketLen); 1288 return MediaChannel::SendRtcp(&packet); 1289} 1290 1291void WebRtcVideoChannel2::StartAllSendStreams() { 1292 rtc::CritScope stream_lock(&stream_crit_); 1293 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1294 send_streams_.begin(); 1295 it != send_streams_.end(); 1296 ++it) { 1297 it->second->Start(); 1298 } 1299} 1300 1301void WebRtcVideoChannel2::StopAllSendStreams() { 1302 rtc::CritScope stream_lock(&stream_crit_); 1303 for (std::map<uint32, WebRtcVideoSendStream*>::iterator it = 1304 send_streams_.begin(); 1305 it != send_streams_.end(); 1306 ++it) { 1307 it->second->Stop(); 1308 } 1309} 1310 1311WebRtcVideoChannel2::WebRtcVideoSendStream::VideoSendStreamParameters:: 1312 VideoSendStreamParameters( 1313 const webrtc::VideoSendStream::Config& config, 1314 const VideoOptions& options, 1315 const Settable<VideoCodecSettings>& codec_settings) 1316 : config(config), options(options), codec_settings(codec_settings) { 1317} 1318 1319WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream( 1320 webrtc::Call* call, 1321 WebRtcVideoEncoderFactory* external_encoder_factory, 1322 const VideoOptions& options, 1323 const Settable<VideoCodecSettings>& codec_settings, 1324 const StreamParams& sp, 1325 const std::vector<webrtc::RtpExtension>& rtp_extensions) 1326 : call_(call), 1327 external_encoder_factory_(external_encoder_factory), 1328 stream_(NULL), 1329 parameters_(webrtc::VideoSendStream::Config(), options, codec_settings), 1330 allocated_encoder_(NULL, webrtc::kVideoCodecUnknown, false), 1331 capturer_(NULL), 1332 sending_(false), 1333 muted_(false) { 1334 parameters_.config.rtp.max_packet_size = kVideoMtu; 1335 1336 sp.GetPrimarySsrcs(¶meters_.config.rtp.ssrcs); 1337 sp.GetFidSsrcs(parameters_.config.rtp.ssrcs, 1338 ¶meters_.config.rtp.rtx.ssrcs); 1339 parameters_.config.rtp.c_name = sp.cname; 1340 parameters_.config.rtp.extensions = rtp_extensions; 1341 1342 VideoCodecSettings params; 1343 if (codec_settings.Get(¶ms)) { 1344 SetCodec(params); 1345 } 1346} 1347 1348WebRtcVideoChannel2::WebRtcVideoSendStream::~WebRtcVideoSendStream() { 1349 DisconnectCapturer(); 1350 if (stream_ != NULL) { 1351 call_->DestroyVideoSendStream(stream_); 1352 } 1353 DestroyVideoEncoder(&allocated_encoder_); 1354} 1355 1356static void CreateBlackFrame(webrtc::I420VideoFrame* video_frame, 1357 int width, 1358 int height) { 1359 video_frame->CreateEmptyFrame(width, height, width, (width + 1) / 2, 1360 (width + 1) / 2); 1361 memset(video_frame->buffer(webrtc::kYPlane), 16, 1362 video_frame->allocated_size(webrtc::kYPlane)); 1363 memset(video_frame->buffer(webrtc::kUPlane), 128, 1364 video_frame->allocated_size(webrtc::kUPlane)); 1365 memset(video_frame->buffer(webrtc::kVPlane), 128, 1366 video_frame->allocated_size(webrtc::kVPlane)); 1367} 1368 1369static void ConvertToI420VideoFrame(const VideoFrame& frame, 1370 webrtc::I420VideoFrame* i420_frame) { 1371 i420_frame->CreateFrame( 1372 static_cast<int>(frame.GetYPitch() * frame.GetHeight()), 1373 frame.GetYPlane(), 1374 static_cast<int>(frame.GetUPitch() * ((frame.GetHeight() + 1) / 2)), 1375 frame.GetUPlane(), 1376 static_cast<int>(frame.GetVPitch() * ((frame.GetHeight() + 1) / 2)), 1377 frame.GetVPlane(), 1378 static_cast<int>(frame.GetWidth()), 1379 static_cast<int>(frame.GetHeight()), 1380 static_cast<int>(frame.GetYPitch()), 1381 static_cast<int>(frame.GetUPitch()), 1382 static_cast<int>(frame.GetVPitch())); 1383} 1384 1385void WebRtcVideoChannel2::WebRtcVideoSendStream::InputFrame( 1386 VideoCapturer* capturer, 1387 const VideoFrame* frame) { 1388 TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::InputFrame"); 1389 LOG(LS_VERBOSE) << "InputFrame: " << frame->GetWidth() << "x" 1390 << frame->GetHeight(); 1391 // Lock before copying, can be called concurrently when swapping input source. 1392 rtc::CritScope frame_cs(&frame_lock_); 1393 ConvertToI420VideoFrame(*frame, &video_frame_); 1394 1395 rtc::CritScope cs(&lock_); 1396 if (stream_ == NULL) { 1397 LOG(LS_WARNING) << "Capturer inputting frames before send codecs are " 1398 "configured, dropping."; 1399 return; 1400 } 1401 1402 // Not sending, abort early to prevent expensive reconfigurations while 1403 // setting up codecs etc. 1404 if (!sending_) 1405 return; 1406 1407 if (format_.width == 0) { // Dropping frames. 1408 assert(format_.height == 0); 1409 LOG(LS_VERBOSE) << "VideoFormat 0x0 set, Dropping frame."; 1410 return; 1411 } 1412 if (muted_) { 1413 // Create a black frame to transmit instead. 1414 CreateBlackFrame(&video_frame_, 1415 static_cast<int>(frame->GetWidth()), 1416 static_cast<int>(frame->GetHeight())); 1417 } 1418 // Reconfigure codec if necessary. 1419 SetDimensions( 1420 video_frame_.width(), video_frame_.height(), capturer->IsScreencast()); 1421 1422 LOG(LS_VERBOSE) << "SwapFrame: " << video_frame_.width() << "x" 1423 << video_frame_.height() << " -> (codec) " 1424 << parameters_.encoder_config.streams.back().width << "x" 1425 << parameters_.encoder_config.streams.back().height; 1426 stream_->Input()->SwapFrame(&video_frame_); 1427} 1428 1429bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetCapturer( 1430 VideoCapturer* capturer) { 1431 TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetCapturer"); 1432 if (!DisconnectCapturer() && capturer == NULL) { 1433 return false; 1434 } 1435 1436 { 1437 rtc::CritScope cs(&lock_); 1438 1439 if (capturer == NULL) { 1440 if (stream_ != NULL) { 1441 LOG(LS_VERBOSE) << "Disabling capturer, sending black frame."; 1442 webrtc::I420VideoFrame black_frame; 1443 1444 CreateBlackFrame(&black_frame, last_dimensions_.width, 1445 last_dimensions_.height); 1446 stream_->Input()->SwapFrame(&black_frame); 1447 } 1448 1449 capturer_ = NULL; 1450 return true; 1451 } 1452 1453 capturer_ = capturer; 1454 } 1455 // Lock cannot be held while connecting the capturer to prevent lock-order 1456 // violations. 1457 capturer->SignalVideoFrame.connect(this, &WebRtcVideoSendStream::InputFrame); 1458 return true; 1459} 1460 1461bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetVideoFormat( 1462 const VideoFormat& format) { 1463 if ((format.width == 0 || format.height == 0) && 1464 format.width != format.height) { 1465 LOG(LS_ERROR) << "Can't set VideoFormat, width or height is zero (but not " 1466 "both, 0x0 drops frames)."; 1467 return false; 1468 } 1469 1470 rtc::CritScope cs(&lock_); 1471 if (format.width == 0 && format.height == 0) { 1472 LOG(LS_INFO) 1473 << "0x0 resolution selected. Captured frames will be dropped for ssrc: " 1474 << parameters_.config.rtp.ssrcs[0] << "."; 1475 } else { 1476 // TODO(pbos): Fix me, this only affects the last stream! 1477 parameters_.encoder_config.streams.back().max_framerate = 1478 VideoFormat::IntervalToFps(format.interval); 1479 SetDimensions(format.width, format.height, false); 1480 } 1481 1482 format_ = format; 1483 return true; 1484} 1485 1486void WebRtcVideoChannel2::WebRtcVideoSendStream::MuteStream(bool mute) { 1487 rtc::CritScope cs(&lock_); 1488 muted_ = mute; 1489} 1490 1491bool WebRtcVideoChannel2::WebRtcVideoSendStream::DisconnectCapturer() { 1492 cricket::VideoCapturer* capturer; 1493 { 1494 rtc::CritScope cs(&lock_); 1495 if (capturer_ == NULL) { 1496 return false; 1497 } 1498 capturer = capturer_; 1499 capturer_ = NULL; 1500 } 1501 capturer->SignalVideoFrame.disconnect(this); 1502 return true; 1503} 1504 1505void WebRtcVideoChannel2::WebRtcVideoSendStream::SetOptions( 1506 const VideoOptions& options) { 1507 rtc::CritScope cs(&lock_); 1508 VideoCodecSettings codec_settings; 1509 if (parameters_.codec_settings.Get(&codec_settings)) { 1510 SetCodecAndOptions(codec_settings, options); 1511 } else { 1512 parameters_.options = options; 1513 } 1514} 1515 1516void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodec( 1517 const VideoCodecSettings& codec_settings) { 1518 rtc::CritScope cs(&lock_); 1519 SetCodecAndOptions(codec_settings, parameters_.options); 1520} 1521 1522webrtc::VideoCodecType CodecTypeFromName(const std::string& name) { 1523 if (CodecNameMatches(name, kVp8CodecName)) { 1524 return webrtc::kVideoCodecVP8; 1525 } else if (CodecNameMatches(name, kVp9CodecName)) { 1526 return webrtc::kVideoCodecVP9; 1527 } else if (CodecNameMatches(name, kH264CodecName)) { 1528 return webrtc::kVideoCodecH264; 1529 } 1530 return webrtc::kVideoCodecUnknown; 1531} 1532 1533WebRtcVideoChannel2::WebRtcVideoSendStream::AllocatedEncoder 1534WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoder( 1535 const VideoCodec& codec) { 1536 webrtc::VideoCodecType type = CodecTypeFromName(codec.name); 1537 1538 // Do not re-create encoders of the same type. 1539 if (type == allocated_encoder_.type && allocated_encoder_.encoder != NULL) { 1540 return allocated_encoder_; 1541 } 1542 1543 if (external_encoder_factory_ != NULL) { 1544 webrtc::VideoEncoder* encoder = 1545 external_encoder_factory_->CreateVideoEncoder(type); 1546 if (encoder != NULL) { 1547 return AllocatedEncoder(encoder, type, true); 1548 } 1549 } 1550 1551 if (type == webrtc::kVideoCodecVP8) { 1552 return AllocatedEncoder( 1553 webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kVp8), type, false); 1554 } else if (type == webrtc::kVideoCodecVP9) { 1555 return AllocatedEncoder( 1556 webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kVp9), type, false); 1557 } 1558 1559 // This shouldn't happen, we should not be trying to create something we don't 1560 // support. 1561 assert(false); 1562 return AllocatedEncoder(NULL, webrtc::kVideoCodecUnknown, false); 1563} 1564 1565void WebRtcVideoChannel2::WebRtcVideoSendStream::DestroyVideoEncoder( 1566 AllocatedEncoder* encoder) { 1567 if (encoder->external) { 1568 external_encoder_factory_->DestroyVideoEncoder(encoder->encoder); 1569 } else { 1570 delete encoder->encoder; 1571 } 1572} 1573 1574void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodecAndOptions( 1575 const VideoCodecSettings& codec_settings, 1576 const VideoOptions& options) { 1577 parameters_.encoder_config = 1578 CreateVideoEncoderConfig(last_dimensions_, codec_settings.codec); 1579 if (parameters_.encoder_config.streams.empty()) 1580 return; 1581 1582 format_ = VideoFormat(codec_settings.codec.width, 1583 codec_settings.codec.height, 1584 VideoFormat::FpsToInterval(30), 1585 FOURCC_I420); 1586 1587 AllocatedEncoder new_encoder = CreateVideoEncoder(codec_settings.codec); 1588 parameters_.config.encoder_settings.encoder = new_encoder.encoder; 1589 parameters_.config.encoder_settings.payload_name = codec_settings.codec.name; 1590 parameters_.config.encoder_settings.payload_type = codec_settings.codec.id; 1591 parameters_.config.rtp.fec = codec_settings.fec; 1592 1593 // Set RTX payload type if RTX is enabled. 1594 if (!parameters_.config.rtp.rtx.ssrcs.empty()) { 1595 parameters_.config.rtp.rtx.payload_type = codec_settings.rtx_payload_type; 1596 } 1597 1598 if (IsNackEnabled(codec_settings.codec)) { 1599 parameters_.config.rtp.nack.rtp_history_ms = kNackHistoryMs; 1600 } 1601 1602 options.suspend_below_min_bitrate.Get( 1603 ¶meters_.config.suspend_below_min_bitrate); 1604 1605 parameters_.codec_settings.Set(codec_settings); 1606 parameters_.options = options; 1607 1608 RecreateWebRtcStream(); 1609 if (allocated_encoder_.encoder != new_encoder.encoder) { 1610 DestroyVideoEncoder(&allocated_encoder_); 1611 allocated_encoder_ = new_encoder; 1612 } 1613} 1614 1615void WebRtcVideoChannel2::WebRtcVideoSendStream::SetRtpExtensions( 1616 const std::vector<webrtc::RtpExtension>& rtp_extensions) { 1617 rtc::CritScope cs(&lock_); 1618 parameters_.config.rtp.extensions = rtp_extensions; 1619 RecreateWebRtcStream(); 1620} 1621 1622webrtc::VideoEncoderConfig 1623WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoderConfig( 1624 const Dimensions& dimensions, 1625 const VideoCodec& codec) const { 1626 webrtc::VideoEncoderConfig encoder_config; 1627 if (dimensions.is_screencast) { 1628 int screencast_min_bitrate_kbps; 1629 parameters_.options.screencast_min_bitrate.Get( 1630 &screencast_min_bitrate_kbps); 1631 encoder_config.min_transmit_bitrate_bps = 1632 screencast_min_bitrate_kbps * 1000; 1633 encoder_config.content_type = webrtc::VideoEncoderConfig::kScreenshare; 1634 } else { 1635 encoder_config.min_transmit_bitrate_bps = 0; 1636 encoder_config.content_type = webrtc::VideoEncoderConfig::kRealtimeVideo; 1637 } 1638 1639 // Restrict dimensions according to codec max. 1640 int width = dimensions.width; 1641 int height = dimensions.height; 1642 if (!dimensions.is_screencast) { 1643 if (codec.width < width) 1644 width = codec.width; 1645 if (codec.height < height) 1646 height = codec.height; 1647 } 1648 1649 VideoCodec clamped_codec = codec; 1650 clamped_codec.width = width; 1651 clamped_codec.height = height; 1652 1653 encoder_config.streams = CreateVideoStreams( 1654 clamped_codec, parameters_.options, parameters_.config.rtp.ssrcs.size()); 1655 1656 // Conference mode screencast uses 2 temporal layers split at 100kbit. 1657 if (parameters_.options.conference_mode.GetWithDefaultIfUnset(false) && 1658 dimensions.is_screencast && encoder_config.streams.size() == 1) { 1659 ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault(); 1660 1661 // For screenshare in conference mode, tl0 and tl1 bitrates are piggybacked 1662 // on the VideoCodec struct as target and max bitrates, respectively. 1663 // See eg. webrtc::VP8EncoderImpl::SetRates(). 1664 encoder_config.streams[0].target_bitrate_bps = 1665 config.tl0_bitrate_kbps * 1000; 1666 encoder_config.streams[0].max_bitrate_bps = config.tl1_bitrate_kbps * 1000; 1667 encoder_config.streams[0].temporal_layer_thresholds_bps.clear(); 1668 encoder_config.streams[0].temporal_layer_thresholds_bps.push_back( 1669 config.tl0_bitrate_kbps * 1000); 1670 } 1671 return encoder_config; 1672} 1673 1674void WebRtcVideoChannel2::WebRtcVideoSendStream::SetDimensions( 1675 int width, 1676 int height, 1677 bool is_screencast) { 1678 if (last_dimensions_.width == width && last_dimensions_.height == height && 1679 last_dimensions_.is_screencast == is_screencast) { 1680 // Configured using the same parameters, do not reconfigure. 1681 return; 1682 } 1683 LOG(LS_INFO) << "SetDimensions: " << width << "x" << height 1684 << (is_screencast ? " (screencast)" : " (not screencast)"); 1685 1686 last_dimensions_.width = width; 1687 last_dimensions_.height = height; 1688 last_dimensions_.is_screencast = is_screencast; 1689 1690 assert(!parameters_.encoder_config.streams.empty()); 1691 1692 VideoCodecSettings codec_settings; 1693 parameters_.codec_settings.Get(&codec_settings); 1694 1695 webrtc::VideoEncoderConfig encoder_config = 1696 CreateVideoEncoderConfig(last_dimensions_, codec_settings.codec); 1697 1698 encoder_config.encoder_specific_settings = 1699 ConfigureVideoEncoderSettings(codec_settings.codec, parameters_.options); 1700 1701 bool stream_reconfigured = stream_->ReconfigureVideoEncoder(encoder_config); 1702 1703 encoder_config.encoder_specific_settings = NULL; 1704 1705 if (!stream_reconfigured) { 1706 LOG(LS_WARNING) << "Failed to reconfigure video encoder for dimensions: " 1707 << width << "x" << height; 1708 return; 1709 } 1710 1711 parameters_.encoder_config = encoder_config; 1712} 1713 1714void WebRtcVideoChannel2::WebRtcVideoSendStream::Start() { 1715 rtc::CritScope cs(&lock_); 1716 assert(stream_ != NULL); 1717 stream_->Start(); 1718 sending_ = true; 1719} 1720 1721void WebRtcVideoChannel2::WebRtcVideoSendStream::Stop() { 1722 rtc::CritScope cs(&lock_); 1723 if (stream_ != NULL) { 1724 stream_->Stop(); 1725 } 1726 sending_ = false; 1727} 1728 1729VideoSenderInfo 1730WebRtcVideoChannel2::WebRtcVideoSendStream::GetVideoSenderInfo() { 1731 VideoSenderInfo info; 1732 webrtc::VideoSendStream::Stats stats; 1733 { 1734 rtc::CritScope cs(&lock_); 1735 for (uint32_t ssrc : parameters_.config.rtp.ssrcs) 1736 info.add_ssrc(ssrc); 1737 1738 for (size_t i = 0; i < parameters_.encoder_config.streams.size(); ++i) { 1739 if (i == parameters_.encoder_config.streams.size() - 1) { 1740 info.preferred_bitrate += 1741 parameters_.encoder_config.streams[i].max_bitrate_bps; 1742 } else { 1743 info.preferred_bitrate += 1744 parameters_.encoder_config.streams[i].target_bitrate_bps; 1745 } 1746 } 1747 1748 if (stream_ == NULL) 1749 return info; 1750 1751 stats = stream_->GetStats(); 1752 1753 if (capturer_ != NULL && !capturer_->IsMuted()) { 1754 VideoFormat last_captured_frame_format; 1755 capturer_->GetStats(&info.adapt_frame_drops, &info.effects_frame_drops, 1756 &info.capturer_frame_time, 1757 &last_captured_frame_format); 1758 info.input_frame_width = last_captured_frame_format.width; 1759 info.input_frame_height = last_captured_frame_format.height; 1760 } 1761 } 1762 info.framerate_input = stats.input_frame_rate; 1763 info.framerate_sent = stats.encode_frame_rate; 1764 1765 info.nominal_bitrate = stats.media_bitrate_bps; 1766 1767 info.send_frame_width = 0; 1768 info.send_frame_height = 0; 1769 for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it = 1770 stats.substreams.begin(); 1771 it != stats.substreams.end(); ++it) { 1772 // TODO(pbos): Wire up additional stats, such as padding bytes. 1773 webrtc::VideoSendStream::StreamStats stream_stats = it->second; 1774 info.bytes_sent += stream_stats.rtp_stats.transmitted.payload_bytes + 1775 stream_stats.rtp_stats.transmitted.header_bytes + 1776 stream_stats.rtp_stats.transmitted.padding_bytes; 1777 info.packets_sent += stream_stats.rtp_stats.transmitted.packets; 1778 info.packets_lost += stream_stats.rtcp_stats.cumulative_lost; 1779 if (stream_stats.width > info.send_frame_width) 1780 info.send_frame_width = stream_stats.width; 1781 if (stream_stats.height > info.send_frame_height) 1782 info.send_frame_height = stream_stats.height; 1783 info.firs_rcvd += stream_stats.rtcp_packet_type_counts.fir_packets; 1784 info.nacks_rcvd += stream_stats.rtcp_packet_type_counts.nack_packets; 1785 info.plis_rcvd += stream_stats.rtcp_packet_type_counts.pli_packets; 1786 } 1787 1788 if (!stats.substreams.empty()) { 1789 // TODO(pbos): Report fraction lost per SSRC. 1790 webrtc::VideoSendStream::StreamStats first_stream_stats = 1791 stats.substreams.begin()->second; 1792 info.fraction_lost = 1793 static_cast<float>(first_stream_stats.rtcp_stats.fraction_lost) / 1794 (1 << 8); 1795 } 1796 1797 return info; 1798} 1799 1800void WebRtcVideoChannel2::WebRtcVideoSendStream::FillBandwidthEstimationInfo( 1801 BandwidthEstimationInfo* bwe_info) { 1802 rtc::CritScope cs(&lock_); 1803 if (stream_ == NULL) { 1804 return; 1805 } 1806 webrtc::VideoSendStream::Stats stats = stream_->GetStats(); 1807 for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it = 1808 stats.substreams.begin(); 1809 it != stats.substreams.end(); ++it) { 1810 bwe_info->transmit_bitrate += it->second.total_bitrate_bps; 1811 bwe_info->retransmit_bitrate += it->second.retransmit_bitrate_bps; 1812 } 1813 bwe_info->actual_enc_bitrate += stats.media_bitrate_bps; 1814} 1815 1816void WebRtcVideoChannel2::WebRtcVideoSendStream::OnCpuResolutionRequest( 1817 CoordinatedVideoAdapter::AdaptRequest adapt_request) { 1818 rtc::CritScope cs(&lock_); 1819 bool adapt_cpu; 1820 parameters_.options.cpu_overuse_detection.Get(&adapt_cpu); 1821 if (!adapt_cpu) { 1822 return; 1823 } 1824 if (capturer_ == NULL || capturer_->video_adapter() == NULL) { 1825 return; 1826 } 1827 1828 capturer_->video_adapter()->OnCpuResolutionRequest(adapt_request); 1829} 1830 1831void WebRtcVideoChannel2::WebRtcVideoSendStream::RecreateWebRtcStream() { 1832 if (stream_ != NULL) { 1833 call_->DestroyVideoSendStream(stream_); 1834 } 1835 1836 VideoCodecSettings codec_settings; 1837 parameters_.codec_settings.Get(&codec_settings); 1838 parameters_.encoder_config.encoder_specific_settings = 1839 ConfigureVideoEncoderSettings(codec_settings.codec, parameters_.options); 1840 1841 stream_ = call_->CreateVideoSendStream(parameters_.config, 1842 parameters_.encoder_config); 1843 1844 parameters_.encoder_config.encoder_specific_settings = NULL; 1845 1846 if (sending_) { 1847 stream_->Start(); 1848 } 1849} 1850 1851WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( 1852 webrtc::Call* call, 1853 WebRtcVideoDecoderFactory* external_decoder_factory, 1854 const webrtc::VideoReceiveStream::Config& config, 1855 const std::vector<VideoCodecSettings>& recv_codecs) 1856 : call_(call), 1857 stream_(NULL), 1858 config_(config), 1859 external_decoder_factory_(external_decoder_factory), 1860 renderer_(NULL), 1861 last_width_(-1), 1862 last_height_(-1), 1863 first_frame_timestamp_(-1), 1864 estimated_remote_start_ntp_time_ms_(0) { 1865 config_.renderer = this; 1866 // SetRecvCodecs will also reset (start) the VideoReceiveStream. 1867 SetRecvCodecs(recv_codecs); 1868} 1869 1870WebRtcVideoChannel2::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() { 1871 call_->DestroyVideoReceiveStream(stream_); 1872 ClearDecoders(&allocated_decoders_); 1873} 1874 1875WebRtcVideoChannel2::WebRtcVideoReceiveStream::AllocatedDecoder 1876WebRtcVideoChannel2::WebRtcVideoReceiveStream::CreateOrReuseVideoDecoder( 1877 std::vector<AllocatedDecoder>* old_decoders, 1878 const VideoCodec& codec) { 1879 webrtc::VideoCodecType type = CodecTypeFromName(codec.name); 1880 1881 for (size_t i = 0; i < old_decoders->size(); ++i) { 1882 if ((*old_decoders)[i].type == type) { 1883 AllocatedDecoder decoder = (*old_decoders)[i]; 1884 (*old_decoders)[i] = old_decoders->back(); 1885 old_decoders->pop_back(); 1886 return decoder; 1887 } 1888 } 1889 1890 if (external_decoder_factory_ != NULL) { 1891 webrtc::VideoDecoder* decoder = 1892 external_decoder_factory_->CreateVideoDecoder(type); 1893 if (decoder != NULL) { 1894 return AllocatedDecoder(decoder, type, true); 1895 } 1896 } 1897 1898 if (type == webrtc::kVideoCodecVP8) { 1899 return AllocatedDecoder( 1900 webrtc::VideoDecoder::Create(webrtc::VideoDecoder::kVp8), type, false); 1901 } 1902 1903 // This shouldn't happen, we should not be trying to create something we don't 1904 // support. 1905 assert(false); 1906 return AllocatedDecoder(NULL, webrtc::kVideoCodecUnknown, false); 1907} 1908 1909void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRecvCodecs( 1910 const std::vector<VideoCodecSettings>& recv_codecs) { 1911 std::vector<AllocatedDecoder> old_decoders = allocated_decoders_; 1912 allocated_decoders_.clear(); 1913 config_.decoders.clear(); 1914 for (size_t i = 0; i < recv_codecs.size(); ++i) { 1915 AllocatedDecoder allocated_decoder = 1916 CreateOrReuseVideoDecoder(&old_decoders, recv_codecs[i].codec); 1917 allocated_decoders_.push_back(allocated_decoder); 1918 1919 webrtc::VideoReceiveStream::Decoder decoder; 1920 decoder.decoder = allocated_decoder.decoder; 1921 decoder.payload_type = recv_codecs[i].codec.id; 1922 decoder.payload_name = recv_codecs[i].codec.name; 1923 config_.decoders.push_back(decoder); 1924 } 1925 1926 // TODO(pbos): Reconfigure RTX based on incoming recv_codecs. 1927 config_.rtp.fec = recv_codecs.front().fec; 1928 config_.rtp.nack.rtp_history_ms = 1929 IsNackEnabled(recv_codecs.begin()->codec) ? kNackHistoryMs : 0; 1930 config_.rtp.remb = IsRembEnabled(recv_codecs.begin()->codec); 1931 1932 ClearDecoders(&old_decoders); 1933 RecreateWebRtcStream(); 1934} 1935 1936void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRtpExtensions( 1937 const std::vector<webrtc::RtpExtension>& extensions) { 1938 config_.rtp.extensions = extensions; 1939 RecreateWebRtcStream(); 1940} 1941 1942void WebRtcVideoChannel2::WebRtcVideoReceiveStream::RecreateWebRtcStream() { 1943 if (stream_ != NULL) { 1944 call_->DestroyVideoReceiveStream(stream_); 1945 } 1946 stream_ = call_->CreateVideoReceiveStream(config_); 1947 stream_->Start(); 1948} 1949 1950void WebRtcVideoChannel2::WebRtcVideoReceiveStream::ClearDecoders( 1951 std::vector<AllocatedDecoder>* allocated_decoders) { 1952 for (size_t i = 0; i < allocated_decoders->size(); ++i) { 1953 if ((*allocated_decoders)[i].external) { 1954 external_decoder_factory_->DestroyVideoDecoder( 1955 (*allocated_decoders)[i].decoder); 1956 } else { 1957 delete (*allocated_decoders)[i].decoder; 1958 } 1959 } 1960 allocated_decoders->clear(); 1961} 1962 1963void WebRtcVideoChannel2::WebRtcVideoReceiveStream::RenderFrame( 1964 const webrtc::I420VideoFrame& frame, 1965 int time_to_render_ms) { 1966 rtc::CritScope crit(&renderer_lock_); 1967 1968 if (first_frame_timestamp_ < 0) 1969 first_frame_timestamp_ = frame.timestamp(); 1970 int64_t rtp_time_elapsed_since_first_frame = 1971 (timestamp_wraparound_handler_.Unwrap(frame.timestamp()) - 1972 first_frame_timestamp_); 1973 int64_t elapsed_time_ms = rtp_time_elapsed_since_first_frame / 1974 (cricket::kVideoCodecClockrate / 1000); 1975 if (frame.ntp_time_ms() > 0) 1976 estimated_remote_start_ntp_time_ms_ = frame.ntp_time_ms() - elapsed_time_ms; 1977 1978 if (renderer_ == NULL) { 1979 LOG(LS_WARNING) << "VideoReceiveStream not connected to a VideoRenderer."; 1980 return; 1981 } 1982 1983 if (frame.width() != last_width_ || frame.height() != last_height_) { 1984 SetSize(frame.width(), frame.height()); 1985 } 1986 1987 LOG(LS_VERBOSE) << "RenderFrame: (" << frame.width() << "x" << frame.height() 1988 << ")"; 1989 1990 const WebRtcVideoRenderFrame render_frame(&frame, elapsed_time_ms); 1991 renderer_->RenderFrame(&render_frame); 1992} 1993 1994bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::IsTextureSupported() const { 1995 return true; 1996} 1997 1998void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRenderer( 1999 cricket::VideoRenderer* renderer) { 2000 rtc::CritScope crit(&renderer_lock_); 2001 renderer_ = renderer; 2002 if (renderer_ != NULL && last_width_ != -1) { 2003 SetSize(last_width_, last_height_); 2004 } 2005} 2006 2007VideoRenderer* WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetRenderer() { 2008 // TODO(pbos): Remove GetRenderer and all uses of it, it's thread-unsafe by 2009 // design. 2010 rtc::CritScope crit(&renderer_lock_); 2011 return renderer_; 2012} 2013 2014void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetSize(int width, 2015 int height) { 2016 rtc::CritScope crit(&renderer_lock_); 2017 if (!renderer_->SetSize(width, height, 0)) { 2018 LOG(LS_ERROR) << "Could not set renderer size."; 2019 } 2020 last_width_ = width; 2021 last_height_ = height; 2022} 2023 2024VideoReceiverInfo 2025WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetVideoReceiverInfo() { 2026 VideoReceiverInfo info; 2027 info.add_ssrc(config_.rtp.remote_ssrc); 2028 webrtc::VideoReceiveStream::Stats stats = stream_->GetStats(); 2029 info.bytes_rcvd = stats.rtp_stats.transmitted.payload_bytes + 2030 stats.rtp_stats.transmitted.header_bytes + 2031 stats.rtp_stats.transmitted.padding_bytes; 2032 info.packets_rcvd = stats.rtp_stats.transmitted.packets; 2033 2034 info.framerate_rcvd = stats.network_frame_rate; 2035 info.framerate_decoded = stats.decode_frame_rate; 2036 info.framerate_output = stats.render_frame_rate; 2037 2038 { 2039 rtc::CritScope frame_cs(&renderer_lock_); 2040 info.frame_width = last_width_; 2041 info.frame_height = last_height_; 2042 info.capture_start_ntp_time_ms = estimated_remote_start_ntp_time_ms_; 2043 } 2044 2045 info.decode_ms = stats.decode_ms; 2046 info.max_decode_ms = stats.max_decode_ms; 2047 info.current_delay_ms = stats.current_delay_ms; 2048 info.target_delay_ms = stats.target_delay_ms; 2049 info.jitter_buffer_ms = stats.jitter_buffer_ms; 2050 info.min_playout_delay_ms = stats.min_playout_delay_ms; 2051 info.render_delay_ms = stats.render_delay_ms; 2052 2053 info.firs_sent = stats.rtcp_packet_type_counts.fir_packets; 2054 info.plis_sent = stats.rtcp_packet_type_counts.pli_packets; 2055 info.nacks_sent = stats.rtcp_packet_type_counts.nack_packets; 2056 2057 return info; 2058} 2059 2060WebRtcVideoChannel2::VideoCodecSettings::VideoCodecSettings() 2061 : rtx_payload_type(-1) {} 2062 2063bool WebRtcVideoChannel2::VideoCodecSettings::operator==( 2064 const WebRtcVideoChannel2::VideoCodecSettings& other) const { 2065 return codec == other.codec && 2066 fec.ulpfec_payload_type == other.fec.ulpfec_payload_type && 2067 fec.red_payload_type == other.fec.red_payload_type && 2068 rtx_payload_type == other.rtx_payload_type; 2069} 2070 2071std::vector<WebRtcVideoChannel2::VideoCodecSettings> 2072WebRtcVideoChannel2::MapCodecs(const std::vector<VideoCodec>& codecs) { 2073 assert(!codecs.empty()); 2074 2075 std::vector<VideoCodecSettings> video_codecs; 2076 std::map<int, bool> payload_used; 2077 std::map<int, VideoCodec::CodecType> payload_codec_type; 2078 // |rtx_mapping| maps video payload type to rtx payload type. 2079 std::map<int, int> rtx_mapping; 2080 2081 webrtc::FecConfig fec_settings; 2082 2083 for (size_t i = 0; i < codecs.size(); ++i) { 2084 const VideoCodec& in_codec = codecs[i]; 2085 int payload_type = in_codec.id; 2086 2087 if (payload_used[payload_type]) { 2088 LOG(LS_ERROR) << "Payload type already registered: " 2089 << in_codec.ToString(); 2090 return std::vector<VideoCodecSettings>(); 2091 } 2092 payload_used[payload_type] = true; 2093 payload_codec_type[payload_type] = in_codec.GetCodecType(); 2094 2095 switch (in_codec.GetCodecType()) { 2096 case VideoCodec::CODEC_RED: { 2097 // RED payload type, should not have duplicates. 2098 assert(fec_settings.red_payload_type == -1); 2099 fec_settings.red_payload_type = in_codec.id; 2100 continue; 2101 } 2102 2103 case VideoCodec::CODEC_ULPFEC: { 2104 // ULPFEC payload type, should not have duplicates. 2105 assert(fec_settings.ulpfec_payload_type == -1); 2106 fec_settings.ulpfec_payload_type = in_codec.id; 2107 continue; 2108 } 2109 2110 case VideoCodec::CODEC_RTX: { 2111 int associated_payload_type; 2112 if (!in_codec.GetParam(kCodecParamAssociatedPayloadType, 2113 &associated_payload_type) || 2114 !IsValidRtpPayloadType(associated_payload_type)) { 2115 LOG(LS_ERROR) 2116 << "RTX codec with invalid or no associated payload type: " 2117 << in_codec.ToString(); 2118 return std::vector<VideoCodecSettings>(); 2119 } 2120 rtx_mapping[associated_payload_type] = in_codec.id; 2121 continue; 2122 } 2123 2124 case VideoCodec::CODEC_VIDEO: 2125 break; 2126 } 2127 2128 video_codecs.push_back(VideoCodecSettings()); 2129 video_codecs.back().codec = in_codec; 2130 } 2131 2132 // One of these codecs should have been a video codec. Only having FEC 2133 // parameters into this code is a logic error. 2134 assert(!video_codecs.empty()); 2135 2136 for (std::map<int, int>::const_iterator it = rtx_mapping.begin(); 2137 it != rtx_mapping.end(); 2138 ++it) { 2139 if (!payload_used[it->first]) { 2140 LOG(LS_ERROR) << "RTX mapped to payload not in codec list."; 2141 return std::vector<VideoCodecSettings>(); 2142 } 2143 if (payload_codec_type[it->first] != VideoCodec::CODEC_VIDEO) { 2144 LOG(LS_ERROR) << "RTX not mapped to regular video codec."; 2145 return std::vector<VideoCodecSettings>(); 2146 } 2147 } 2148 2149 // TODO(pbos): Write tests that figure out that I have not verified that RTX 2150 // codecs aren't mapped to bogus payloads. 2151 for (size_t i = 0; i < video_codecs.size(); ++i) { 2152 video_codecs[i].fec = fec_settings; 2153 if (rtx_mapping[video_codecs[i].codec.id] != 0) { 2154 video_codecs[i].rtx_payload_type = rtx_mapping[video_codecs[i].codec.id]; 2155 } 2156 } 2157 2158 return video_codecs; 2159} 2160 2161} // namespace cricket 2162 2163#endif // HAVE_WEBRTC_VIDEO 2164