1/* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#include "webrtc/common_types.h" 12#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" 13#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" 14#include "webrtc/modules/video_coding/main/source/encoded_frame.h" 15#include "webrtc/modules/video_coding/main/source/jitter_buffer.h" 16#include "webrtc/modules/video_coding/main/source/packet.h" 17#include "webrtc/modules/video_coding/main/source/video_coding_impl.h" 18#include "webrtc/system_wrappers/interface/clock.h" 19#include "webrtc/system_wrappers/interface/logging.h" 20#include "webrtc/system_wrappers/interface/trace_event.h" 21 22// #define DEBUG_DECODER_BIT_STREAM 23 24namespace webrtc { 25namespace vcm { 26 27VideoReceiver::VideoReceiver(Clock* clock, EventFactory* event_factory) 28 : clock_(clock), 29 process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), 30 _receiveCritSect(CriticalSectionWrapper::CreateCriticalSection()), 31 _receiverInited(false), 32 _timing(clock_), 33 _dualTiming(clock_, &_timing), 34 _receiver(&_timing, clock_, event_factory, true), 35 _dualReceiver(&_dualTiming, clock_, event_factory, false), 36 _decodedFrameCallback(_timing, clock_), 37 _dualDecodedFrameCallback(_dualTiming, clock_), 38 _frameTypeCallback(NULL), 39 _receiveStatsCallback(NULL), 40 _decoderTimingCallback(NULL), 41 _packetRequestCallback(NULL), 42 render_buffer_callback_(NULL), 43 _decoder(NULL), 44 _dualDecoder(NULL), 45#ifdef DEBUG_DECODER_BIT_STREAM 46 _bitStreamBeforeDecoder(NULL), 47#endif 48 _frameFromFile(), 49 _keyRequestMode(kKeyOnError), 50 _scheduleKeyRequest(false), 51 max_nack_list_size_(0), 52 pre_decode_image_callback_(NULL), 53 _codecDataBase(), 54 _receiveStatsTimer(1000, clock_), 55 _retransmissionTimer(10, clock_), 56 _keyRequestTimer(500, clock_) { 57 assert(clock_); 58#ifdef DEBUG_DECODER_BIT_STREAM 59 _bitStreamBeforeDecoder = fopen("decoderBitStream.bit", "wb"); 60#endif 61} 62 63VideoReceiver::~VideoReceiver() { 64 if (_dualDecoder != NULL) { 65 _codecDataBase.ReleaseDecoder(_dualDecoder); 66 } 67 delete _receiveCritSect; 68#ifdef DEBUG_DECODER_BIT_STREAM 69 fclose(_bitStreamBeforeDecoder); 70#endif 71} 72 73int32_t VideoReceiver::Process() { 74 int32_t returnValue = VCM_OK; 75 76 // Receive-side statistics 77 if (_receiveStatsTimer.TimeUntilProcess() == 0) { 78 _receiveStatsTimer.Processed(); 79 CriticalSectionScoped cs(process_crit_sect_.get()); 80 if (_receiveStatsCallback != NULL) { 81 uint32_t bitRate; 82 uint32_t frameRate; 83 _receiver.ReceiveStatistics(&bitRate, &frameRate); 84 _receiveStatsCallback->OnReceiveStatisticsUpdate(bitRate, frameRate); 85 } 86 87 if (_decoderTimingCallback != NULL) { 88 int decode_ms; 89 int max_decode_ms; 90 int current_delay_ms; 91 int target_delay_ms; 92 int jitter_buffer_ms; 93 int min_playout_delay_ms; 94 int render_delay_ms; 95 _timing.GetTimings(&decode_ms, 96 &max_decode_ms, 97 ¤t_delay_ms, 98 &target_delay_ms, 99 &jitter_buffer_ms, 100 &min_playout_delay_ms, 101 &render_delay_ms); 102 _decoderTimingCallback->OnDecoderTiming(decode_ms, 103 max_decode_ms, 104 current_delay_ms, 105 target_delay_ms, 106 jitter_buffer_ms, 107 min_playout_delay_ms, 108 render_delay_ms); 109 } 110 111 // Size of render buffer. 112 if (render_buffer_callback_) { 113 int buffer_size_ms = _receiver.RenderBufferSizeMs(); 114 render_buffer_callback_->RenderBufferSizeMs(buffer_size_ms); 115 } 116 } 117 118 // Key frame requests 119 if (_keyRequestTimer.TimeUntilProcess() == 0) { 120 _keyRequestTimer.Processed(); 121 bool request_key_frame = false; 122 { 123 CriticalSectionScoped cs(process_crit_sect_.get()); 124 request_key_frame = _scheduleKeyRequest && _frameTypeCallback != NULL; 125 } 126 if (request_key_frame) { 127 const int32_t ret = RequestKeyFrame(); 128 if (ret != VCM_OK && returnValue == VCM_OK) { 129 returnValue = ret; 130 } 131 } 132 } 133 134 // Packet retransmission requests 135 // TODO(holmer): Add API for changing Process interval and make sure it's 136 // disabled when NACK is off. 137 if (_retransmissionTimer.TimeUntilProcess() == 0) { 138 _retransmissionTimer.Processed(); 139 bool callback_registered = false; 140 uint16_t length; 141 { 142 CriticalSectionScoped cs(process_crit_sect_.get()); 143 length = max_nack_list_size_; 144 callback_registered = _packetRequestCallback != NULL; 145 } 146 if (callback_registered && length > 0) { 147 std::vector<uint16_t> nackList(length); 148 const int32_t ret = NackList(&nackList[0], &length); 149 if (ret != VCM_OK && returnValue == VCM_OK) { 150 returnValue = ret; 151 } 152 if (ret == VCM_OK && length > 0) { 153 CriticalSectionScoped cs(process_crit_sect_.get()); 154 if (_packetRequestCallback != NULL) { 155 _packetRequestCallback->ResendPackets(&nackList[0], length); 156 } 157 } 158 } 159 } 160 161 return returnValue; 162} 163 164int32_t VideoReceiver::TimeUntilNextProcess() { 165 uint32_t timeUntilNextProcess = _receiveStatsTimer.TimeUntilProcess(); 166 if ((_receiver.NackMode() != kNoNack) || 167 (_dualReceiver.State() != kPassive)) { 168 // We need a Process call more often if we are relying on 169 // retransmissions 170 timeUntilNextProcess = 171 VCM_MIN(timeUntilNextProcess, _retransmissionTimer.TimeUntilProcess()); 172 } 173 timeUntilNextProcess = 174 VCM_MIN(timeUntilNextProcess, _keyRequestTimer.TimeUntilProcess()); 175 176 return timeUntilNextProcess; 177} 178 179int32_t VideoReceiver::SetReceiveChannelParameters(uint32_t rtt) { 180 CriticalSectionScoped receiveCs(_receiveCritSect); 181 _receiver.UpdateRtt(rtt); 182 return 0; 183} 184 185// Enable or disable a video protection method. 186// Note: This API should be deprecated, as it does not offer a distinction 187// between the protection method and decoding with or without errors. If such a 188// behavior is desired, use the following API: SetReceiverRobustnessMode. 189int32_t VideoReceiver::SetVideoProtection(VCMVideoProtection videoProtection, 190 bool enable) { 191 // By default, do not decode with errors. 192 _receiver.SetDecodeErrorMode(kNoErrors); 193 // The dual decoder should always be error free. 194 _dualReceiver.SetDecodeErrorMode(kNoErrors); 195 switch (videoProtection) { 196 case kProtectionNack: 197 case kProtectionNackReceiver: { 198 CriticalSectionScoped cs(_receiveCritSect); 199 if (enable) { 200 // Enable NACK and always wait for retransmits. 201 _receiver.SetNackMode(kNack, -1, -1); 202 } else { 203 _receiver.SetNackMode(kNoNack, -1, -1); 204 } 205 break; 206 } 207 208 case kProtectionDualDecoder: { 209 CriticalSectionScoped cs(_receiveCritSect); 210 if (enable) { 211 // Enable NACK but don't wait for retransmissions and don't 212 // add any extra delay. 213 _receiver.SetNackMode(kNack, 0, 0); 214 // Enable NACK and always wait for retransmissions and 215 // compensate with extra delay. 216 _dualReceiver.SetNackMode(kNack, -1, -1); 217 _receiver.SetDecodeErrorMode(kWithErrors); 218 } else { 219 _dualReceiver.SetNackMode(kNoNack, -1, -1); 220 } 221 break; 222 } 223 224 case kProtectionKeyOnLoss: { 225 CriticalSectionScoped cs(_receiveCritSect); 226 if (enable) { 227 _keyRequestMode = kKeyOnLoss; 228 _receiver.SetDecodeErrorMode(kWithErrors); 229 } else if (_keyRequestMode == kKeyOnLoss) { 230 _keyRequestMode = kKeyOnError; // default mode 231 } else { 232 return VCM_PARAMETER_ERROR; 233 } 234 break; 235 } 236 237 case kProtectionKeyOnKeyLoss: { 238 CriticalSectionScoped cs(_receiveCritSect); 239 if (enable) { 240 _keyRequestMode = kKeyOnKeyLoss; 241 } else if (_keyRequestMode == kKeyOnKeyLoss) { 242 _keyRequestMode = kKeyOnError; // default mode 243 } else { 244 return VCM_PARAMETER_ERROR; 245 } 246 break; 247 } 248 249 case kProtectionNackFEC: { 250 CriticalSectionScoped cs(_receiveCritSect); 251 if (enable) { 252 // Enable hybrid NACK/FEC. Always wait for retransmissions 253 // and don't add extra delay when RTT is above 254 // kLowRttNackMs. 255 _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1); 256 _receiver.SetDecodeErrorMode(kNoErrors); 257 _receiver.SetDecodeErrorMode(kNoErrors); 258 } else { 259 _receiver.SetNackMode(kNoNack, -1, -1); 260 } 261 break; 262 } 263 case kProtectionNackSender: 264 case kProtectionFEC: 265 case kProtectionPeriodicKeyFrames: 266 // Ignore encoder modes. 267 return VCM_OK; 268 } 269 return VCM_OK; 270} 271 272// Initialize receiver, resets codec database etc 273int32_t VideoReceiver::InitializeReceiver() { 274 int32_t ret = _receiver.Initialize(); 275 if (ret < 0) { 276 return ret; 277 } 278 279 ret = _dualReceiver.Initialize(); 280 if (ret < 0) { 281 return ret; 282 } 283 _codecDataBase.ResetReceiver(); 284 _timing.Reset(); 285 286 { 287 CriticalSectionScoped receive_cs(_receiveCritSect); 288 _receiverInited = true; 289 } 290 291 { 292 CriticalSectionScoped process_cs(process_crit_sect_.get()); 293 _decoder = NULL; 294 _decodedFrameCallback.SetUserReceiveCallback(NULL); 295 _frameTypeCallback = NULL; 296 _receiveStatsCallback = NULL; 297 _decoderTimingCallback = NULL; 298 _packetRequestCallback = NULL; 299 _keyRequestMode = kKeyOnError; 300 _scheduleKeyRequest = false; 301 } 302 303 return VCM_OK; 304} 305 306// Register a receive callback. Will be called whenever there is a new frame 307// ready for rendering. 308int32_t VideoReceiver::RegisterReceiveCallback( 309 VCMReceiveCallback* receiveCallback) { 310 CriticalSectionScoped cs(_receiveCritSect); 311 _decodedFrameCallback.SetUserReceiveCallback(receiveCallback); 312 return VCM_OK; 313} 314 315int32_t VideoReceiver::RegisterReceiveStatisticsCallback( 316 VCMReceiveStatisticsCallback* receiveStats) { 317 CriticalSectionScoped cs(process_crit_sect_.get()); 318 _receiveStatsCallback = receiveStats; 319 return VCM_OK; 320} 321 322int32_t VideoReceiver::RegisterDecoderTimingCallback( 323 VCMDecoderTimingCallback* decoderTiming) { 324 CriticalSectionScoped cs(process_crit_sect_.get()); 325 _decoderTimingCallback = decoderTiming; 326 return VCM_OK; 327} 328 329// Register an externally defined decoder/render object. 330// Can be a decoder only or a decoder coupled with a renderer. 331int32_t VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder, 332 uint8_t payloadType, 333 bool internalRenderTiming) { 334 CriticalSectionScoped cs(_receiveCritSect); 335 if (externalDecoder == NULL) { 336 // Make sure the VCM updates the decoder next time it decodes. 337 _decoder = NULL; 338 return _codecDataBase.DeregisterExternalDecoder(payloadType) ? 0 : -1; 339 } 340 return _codecDataBase.RegisterExternalDecoder( 341 externalDecoder, payloadType, internalRenderTiming) 342 ? 0 343 : -1; 344} 345 346// Register a frame type request callback. 347int32_t VideoReceiver::RegisterFrameTypeCallback( 348 VCMFrameTypeCallback* frameTypeCallback) { 349 CriticalSectionScoped cs(process_crit_sect_.get()); 350 _frameTypeCallback = frameTypeCallback; 351 return VCM_OK; 352} 353 354int32_t VideoReceiver::RegisterPacketRequestCallback( 355 VCMPacketRequestCallback* callback) { 356 CriticalSectionScoped cs(process_crit_sect_.get()); 357 _packetRequestCallback = callback; 358 return VCM_OK; 359} 360 361int VideoReceiver::RegisterRenderBufferSizeCallback( 362 VCMRenderBufferSizeCallback* callback) { 363 CriticalSectionScoped cs(process_crit_sect_.get()); 364 render_buffer_callback_ = callback; 365 return VCM_OK; 366} 367 368// Decode next frame, blocking. 369// Should be called as often as possible to get the most out of the decoder. 370int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) { 371 int64_t nextRenderTimeMs; 372 { 373 CriticalSectionScoped cs(_receiveCritSect); 374 if (!_receiverInited) { 375 return VCM_UNINITIALIZED; 376 } 377 if (!_codecDataBase.DecoderRegistered()) { 378 return VCM_NO_CODEC_REGISTERED; 379 } 380 } 381 382 const bool dualReceiverEnabledNotReceiving = ( 383 _dualReceiver.State() != kReceiving && _dualReceiver.NackMode() == kNack); 384 385 VCMEncodedFrame* frame = 386 _receiver.FrameForDecoding(maxWaitTimeMs, 387 nextRenderTimeMs, 388 _codecDataBase.SupportsRenderScheduling(), 389 &_dualReceiver); 390 391 if (dualReceiverEnabledNotReceiving && _dualReceiver.State() == kReceiving) { 392 // Dual receiver is enabled (kNACK enabled), but was not receiving 393 // before the call to FrameForDecoding(). After the call the state 394 // changed to receiving, and therefore we must copy the primary decoder 395 // state to the dual decoder to make it possible for the dual decoder to 396 // start decoding retransmitted frames and recover. 397 CriticalSectionScoped cs(_receiveCritSect); 398 if (_dualDecoder != NULL) { 399 _codecDataBase.ReleaseDecoder(_dualDecoder); 400 } 401 _dualDecoder = _codecDataBase.CreateDecoderCopy(); 402 if (_dualDecoder != NULL) { 403 _dualDecoder->RegisterDecodeCompleteCallback(&_dualDecodedFrameCallback); 404 } else { 405 _dualReceiver.Reset(); 406 } 407 } 408 409 if (frame == NULL) { 410 return VCM_FRAME_NOT_READY; 411 } else { 412 CriticalSectionScoped cs(_receiveCritSect); 413 414 // If this frame was too late, we should adjust the delay accordingly 415 _timing.UpdateCurrentDelay(frame->RenderTimeMs(), 416 clock_->TimeInMilliseconds()); 417 418 if (pre_decode_image_callback_) { 419 EncodedImage encoded_image(frame->EncodedImage()); 420 pre_decode_image_callback_->Encoded(encoded_image); 421 } 422 423#ifdef DEBUG_DECODER_BIT_STREAM 424 if (_bitStreamBeforeDecoder != NULL) { 425 // Write bit stream to file for debugging purposes 426 if (fwrite( 427 frame->Buffer(), 1, frame->Length(), _bitStreamBeforeDecoder) != 428 frame->Length()) { 429 return -1; 430 } 431 } 432#endif 433 const int32_t ret = Decode(*frame); 434 _receiver.ReleaseFrame(frame); 435 frame = NULL; 436 if (ret != VCM_OK) { 437 return ret; 438 } 439 } 440 return VCM_OK; 441} 442 443int32_t VideoReceiver::RequestSliceLossIndication( 444 const uint64_t pictureID) const { 445 TRACE_EVENT1("webrtc", "RequestSLI", "picture_id", pictureID); 446 CriticalSectionScoped cs(process_crit_sect_.get()); 447 if (_frameTypeCallback != NULL) { 448 const int32_t ret = 449 _frameTypeCallback->SliceLossIndicationRequest(pictureID); 450 if (ret < 0) { 451 return ret; 452 } 453 } else { 454 return VCM_MISSING_CALLBACK; 455 } 456 return VCM_OK; 457} 458 459int32_t VideoReceiver::RequestKeyFrame() { 460 TRACE_EVENT0("webrtc", "RequestKeyFrame"); 461 CriticalSectionScoped process_cs(process_crit_sect_.get()); 462 if (_frameTypeCallback != NULL) { 463 const int32_t ret = _frameTypeCallback->RequestKeyFrame(); 464 if (ret < 0) { 465 return ret; 466 } 467 _scheduleKeyRequest = false; 468 } else { 469 return VCM_MISSING_CALLBACK; 470 } 471 return VCM_OK; 472} 473 474int32_t VideoReceiver::DecodeDualFrame(uint16_t maxWaitTimeMs) { 475 CriticalSectionScoped cs(_receiveCritSect); 476 if (_dualReceiver.State() != kReceiving || 477 _dualReceiver.NackMode() != kNack) { 478 // The dual receiver is currently not receiving or 479 // dual decoder mode is disabled. 480 return VCM_OK; 481 } 482 int64_t dummyRenderTime; 483 int32_t decodeCount = 0; 484 // The dual decoder's state is copied from the main decoder, which may 485 // decode with errors. Make sure that the dual decoder does not introduce 486 // error. 487 _dualReceiver.SetDecodeErrorMode(kNoErrors); 488 VCMEncodedFrame* dualFrame = 489 _dualReceiver.FrameForDecoding(maxWaitTimeMs, dummyRenderTime); 490 if (dualFrame != NULL && _dualDecoder != NULL) { 491 // Decode dualFrame and try to catch up 492 int32_t ret = 493 _dualDecoder->Decode(*dualFrame, clock_->TimeInMilliseconds()); 494 if (ret != WEBRTC_VIDEO_CODEC_OK) { 495 LOG(LS_ERROR) << "Failed to decode frame with dual decoder. Error code: " 496 << ret; 497 _dualReceiver.ReleaseFrame(dualFrame); 498 return VCM_CODEC_ERROR; 499 } 500 if (_receiver.DualDecoderCaughtUp(dualFrame, _dualReceiver)) { 501 // Copy the complete decoder state of the dual decoder 502 // to the primary decoder. 503 _codecDataBase.CopyDecoder(*_dualDecoder); 504 _codecDataBase.ReleaseDecoder(_dualDecoder); 505 _dualDecoder = NULL; 506 } 507 decodeCount++; 508 } 509 _dualReceiver.ReleaseFrame(dualFrame); 510 return decodeCount; 511} 512 513// Must be called from inside the receive side critical section. 514int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) { 515 TRACE_EVENT_ASYNC_STEP1("webrtc", 516 "Video", 517 frame.TimeStamp(), 518 "Decode", 519 "type", 520 frame.FrameType()); 521 // Change decoder if payload type has changed 522 const bool renderTimingBefore = _codecDataBase.SupportsRenderScheduling(); 523 _decoder = 524 _codecDataBase.GetDecoder(frame.PayloadType(), &_decodedFrameCallback); 525 if (renderTimingBefore != _codecDataBase.SupportsRenderScheduling()) { 526 // Make sure we reset the decode time estimate since it will 527 // be zero for codecs without render timing. 528 _timing.ResetDecodeTime(); 529 } 530 if (_decoder == NULL) { 531 return VCM_NO_CODEC_REGISTERED; 532 } 533 // Decode a frame 534 int32_t ret = _decoder->Decode(frame, clock_->TimeInMilliseconds()); 535 536 // Check for failed decoding, run frame type request callback if needed. 537 bool request_key_frame = false; 538 if (ret < 0) { 539 if (ret == VCM_ERROR_REQUEST_SLI) { 540 return RequestSliceLossIndication( 541 _decodedFrameCallback.LastReceivedPictureID() + 1); 542 } else { 543 request_key_frame = true; 544 } 545 } else if (ret == VCM_REQUEST_SLI) { 546 ret = RequestSliceLossIndication( 547 _decodedFrameCallback.LastReceivedPictureID() + 1); 548 } 549 if (!frame.Complete() || frame.MissingFrame()) { 550 switch (_keyRequestMode) { 551 case kKeyOnKeyLoss: { 552 if (frame.FrameType() == kVideoFrameKey) { 553 request_key_frame = true; 554 ret = VCM_OK; 555 } 556 break; 557 } 558 case kKeyOnLoss: { 559 request_key_frame = true; 560 ret = VCM_OK; 561 } 562 default: 563 break; 564 } 565 } 566 if (request_key_frame) { 567 CriticalSectionScoped cs(process_crit_sect_.get()); 568 _scheduleKeyRequest = true; 569 } 570 TRACE_EVENT_ASYNC_END0("webrtc", "Video", frame.TimeStamp()); 571 return ret; 572} 573 574// Reset the decoder state 575int32_t VideoReceiver::ResetDecoder() { 576 bool reset_key_request = false; 577 { 578 CriticalSectionScoped cs(_receiveCritSect); 579 if (_decoder != NULL) { 580 _receiver.Initialize(); 581 _timing.Reset(); 582 reset_key_request = true; 583 _decoder->Reset(); 584 } 585 if (_dualReceiver.State() != kPassive) { 586 _dualReceiver.Initialize(); 587 } 588 if (_dualDecoder != NULL) { 589 _codecDataBase.ReleaseDecoder(_dualDecoder); 590 _dualDecoder = NULL; 591 } 592 } 593 if (reset_key_request) { 594 CriticalSectionScoped cs(process_crit_sect_.get()); 595 _scheduleKeyRequest = false; 596 } 597 return VCM_OK; 598} 599 600// Register possible receive codecs, can be called multiple times 601int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec, 602 int32_t numberOfCores, 603 bool requireKeyFrame) { 604 CriticalSectionScoped cs(_receiveCritSect); 605 if (receiveCodec == NULL) { 606 return VCM_PARAMETER_ERROR; 607 } 608 if (!_codecDataBase.RegisterReceiveCodec( 609 receiveCodec, numberOfCores, requireKeyFrame)) { 610 return -1; 611 } 612 return 0; 613} 614 615// Get current received codec 616int32_t VideoReceiver::ReceiveCodec(VideoCodec* currentReceiveCodec) const { 617 CriticalSectionScoped cs(_receiveCritSect); 618 if (currentReceiveCodec == NULL) { 619 return VCM_PARAMETER_ERROR; 620 } 621 return _codecDataBase.ReceiveCodec(currentReceiveCodec) ? 0 : -1; 622} 623 624// Get current received codec 625VideoCodecType VideoReceiver::ReceiveCodec() const { 626 CriticalSectionScoped cs(_receiveCritSect); 627 return _codecDataBase.ReceiveCodec(); 628} 629 630// Incoming packet from network parsed and ready for decode, non blocking. 631int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload, 632 uint32_t payloadLength, 633 const WebRtcRTPHeader& rtpInfo) { 634 if (rtpInfo.frameType == kVideoFrameKey) { 635 TRACE_EVENT1("webrtc", 636 "VCM::PacketKeyFrame", 637 "seqnum", 638 rtpInfo.header.sequenceNumber); 639 } 640 if (incomingPayload == NULL) { 641 // The jitter buffer doesn't handle non-zero payload lengths for packets 642 // without payload. 643 // TODO(holmer): We should fix this in the jitter buffer. 644 payloadLength = 0; 645 } 646 const VCMPacket packet(incomingPayload, payloadLength, rtpInfo); 647 int32_t ret; 648 if (_dualReceiver.State() != kPassive) { 649 ret = _dualReceiver.InsertPacket( 650 packet, rtpInfo.type.Video.width, rtpInfo.type.Video.height); 651 if (ret == VCM_FLUSH_INDICATOR) { 652 RequestKeyFrame(); 653 ResetDecoder(); 654 } else if (ret < 0) { 655 return ret; 656 } 657 } 658 ret = _receiver.InsertPacket( 659 packet, rtpInfo.type.Video.width, rtpInfo.type.Video.height); 660 // TODO(holmer): Investigate if this somehow should use the key frame 661 // request scheduling to throttle the requests. 662 if (ret == VCM_FLUSH_INDICATOR) { 663 RequestKeyFrame(); 664 ResetDecoder(); 665 } else if (ret < 0) { 666 return ret; 667 } 668 return VCM_OK; 669} 670 671// Minimum playout delay (used for lip-sync). This is the minimum delay required 672// to sync with audio. Not included in VideoCodingModule::Delay() 673// Defaults to 0 ms. 674int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) { 675 _timing.set_min_playout_delay(minPlayoutDelayMs); 676 return VCM_OK; 677} 678 679// The estimated delay caused by rendering, defaults to 680// kDefaultRenderDelayMs = 10 ms 681int32_t VideoReceiver::SetRenderDelay(uint32_t timeMS) { 682 _timing.set_render_delay(timeMS); 683 return VCM_OK; 684} 685 686// Current video delay 687int32_t VideoReceiver::Delay() const { return _timing.TargetVideoDelay(); } 688 689// Nack list 690int32_t VideoReceiver::NackList(uint16_t* nackList, uint16_t* size) { 691 VCMNackStatus nackStatus = kNackOk; 692 uint16_t nack_list_length = 0; 693 // Collect sequence numbers from the default receiver 694 // if in normal nack mode. Otherwise collect them from 695 // the dual receiver if the dual receiver is receiving. 696 if (_receiver.NackMode() != kNoNack) { 697 nackStatus = _receiver.NackList(nackList, *size, &nack_list_length); 698 } 699 if (nack_list_length == 0 && _dualReceiver.State() != kPassive) { 700 nackStatus = _dualReceiver.NackList(nackList, *size, &nack_list_length); 701 } 702 *size = nack_list_length; 703 if (nackStatus == kNackKeyFrameRequest) { 704 return RequestKeyFrame(); 705 } 706 return VCM_OK; 707} 708 709int32_t VideoReceiver::ReceivedFrameCount(VCMFrameCount* frameCount) const { 710 _receiver.ReceivedFrameCount(frameCount); 711 return VCM_OK; 712} 713 714uint32_t VideoReceiver::DiscardedPackets() const { 715 return _receiver.DiscardedPackets(); 716} 717 718int VideoReceiver::SetReceiverRobustnessMode( 719 ReceiverRobustness robustnessMode, 720 VCMDecodeErrorMode decode_error_mode) { 721 CriticalSectionScoped cs(_receiveCritSect); 722 switch (robustnessMode) { 723 case VideoCodingModule::kNone: 724 _receiver.SetNackMode(kNoNack, -1, -1); 725 _dualReceiver.SetNackMode(kNoNack, -1, -1); 726 if (decode_error_mode == kNoErrors) { 727 _keyRequestMode = kKeyOnLoss; 728 } else { 729 _keyRequestMode = kKeyOnError; 730 } 731 break; 732 case VideoCodingModule::kHardNack: 733 // Always wait for retransmissions (except when decoding with errors). 734 _receiver.SetNackMode(kNack, -1, -1); 735 _dualReceiver.SetNackMode(kNoNack, -1, -1); 736 _keyRequestMode = kKeyOnError; // TODO(hlundin): On long NACK list? 737 break; 738 case VideoCodingModule::kSoftNack: 739#if 1 740 assert(false); // TODO(hlundin): Not completed. 741 return VCM_NOT_IMPLEMENTED; 742#else 743 // Enable hybrid NACK/FEC. Always wait for retransmissions and don't add 744 // extra delay when RTT is above kLowRttNackMs. 745 _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1); 746 _dualReceiver.SetNackMode(kNoNack, -1, -1); 747 _keyRequestMode = kKeyOnError; 748 break; 749#endif 750 case VideoCodingModule::kDualDecoder: 751 if (decode_error_mode == kNoErrors) { 752 return VCM_PARAMETER_ERROR; 753 } 754 // Enable NACK but don't wait for retransmissions and don't add any extra 755 // delay. 756 _receiver.SetNackMode(kNack, 0, 0); 757 // Enable NACK, compensate with extra delay and wait for retransmissions. 758 _dualReceiver.SetNackMode(kNack, -1, -1); 759 _keyRequestMode = kKeyOnError; 760 break; 761 case VideoCodingModule::kReferenceSelection: 762#if 1 763 assert(false); // TODO(hlundin): Not completed. 764 return VCM_NOT_IMPLEMENTED; 765#else 766 if (decode_error_mode == kNoErrors) { 767 return VCM_PARAMETER_ERROR; 768 } 769 _receiver.SetNackMode(kNoNack, -1, -1); 770 _dualReceiver.SetNackMode(kNoNack, -1, -1); 771 break; 772#endif 773 } 774 _receiver.SetDecodeErrorMode(decode_error_mode); 775 // The dual decoder should never decode with errors. 776 _dualReceiver.SetDecodeErrorMode(kNoErrors); 777 return VCM_OK; 778} 779 780void VideoReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) { 781 CriticalSectionScoped cs(_receiveCritSect); 782 _receiver.SetDecodeErrorMode(decode_error_mode); 783} 784 785void VideoReceiver::SetNackSettings(size_t max_nack_list_size, 786 int max_packet_age_to_nack, 787 int max_incomplete_time_ms) { 788 if (max_nack_list_size != 0) { 789 CriticalSectionScoped process_cs(process_crit_sect_.get()); 790 max_nack_list_size_ = max_nack_list_size; 791 } 792 _receiver.SetNackSettings( 793 max_nack_list_size, max_packet_age_to_nack, max_incomplete_time_ms); 794 _dualReceiver.SetNackSettings( 795 max_nack_list_size, max_packet_age_to_nack, max_incomplete_time_ms); 796} 797 798int VideoReceiver::SetMinReceiverDelay(int desired_delay_ms) { 799 return _receiver.SetMinReceiverDelay(desired_delay_ms); 800} 801 802void VideoReceiver::RegisterPreDecodeImageCallback( 803 EncodedImageCallback* observer) { 804 CriticalSectionScoped cs(_receiveCritSect); 805 pre_decode_image_callback_ = observer; 806} 807 808} // namespace vcm 809} // namespace webrtc 810