1/* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#include "webrtc/base/checks.h" 12#include "webrtc/base/logging.h" 13#include "webrtc/base/trace_event.h" 14#include "webrtc/common_types.h" 15#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" 16#include "webrtc/modules/video_coding/include/video_codec_interface.h" 17#include "webrtc/modules/video_coding/encoded_frame.h" 18#include "webrtc/modules/video_coding/jitter_buffer.h" 19#include "webrtc/modules/video_coding/packet.h" 20#include "webrtc/modules/video_coding/video_coding_impl.h" 21#include "webrtc/system_wrappers/include/clock.h" 22 23// #define DEBUG_DECODER_BIT_STREAM 24 25namespace webrtc { 26namespace vcm { 27 28VideoReceiver::VideoReceiver(Clock* clock, EventFactory* event_factory) 29 : clock_(clock), 30 process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), 31 _receiveCritSect(CriticalSectionWrapper::CreateCriticalSection()), 32 _timing(clock_), 33 _receiver(&_timing, clock_, event_factory), 34 _decodedFrameCallback(&_timing, clock_), 35 _frameTypeCallback(NULL), 36 _receiveStatsCallback(NULL), 37 _decoderTimingCallback(NULL), 38 _packetRequestCallback(NULL), 39 render_buffer_callback_(NULL), 40 _decoder(NULL), 41#ifdef DEBUG_DECODER_BIT_STREAM 42 _bitStreamBeforeDecoder(NULL), 43#endif 44 _frameFromFile(), 45 _scheduleKeyRequest(false), 46 max_nack_list_size_(0), 47 pre_decode_image_callback_(NULL), 48 _codecDataBase(nullptr, nullptr), 49 _receiveStatsTimer(1000, clock_), 50 _retransmissionTimer(10, clock_), 51 _keyRequestTimer(500, clock_) { 52 assert(clock_); 53#ifdef DEBUG_DECODER_BIT_STREAM 54 _bitStreamBeforeDecoder = fopen("decoderBitStream.bit", "wb"); 55#endif 56} 57 58VideoReceiver::~VideoReceiver() { 59 delete _receiveCritSect; 60#ifdef DEBUG_DECODER_BIT_STREAM 61 fclose(_bitStreamBeforeDecoder); 62#endif 63} 64 65int32_t VideoReceiver::Process() { 66 int32_t returnValue = VCM_OK; 67 68 // Receive-side statistics 69 if (_receiveStatsTimer.TimeUntilProcess() == 0) { 70 _receiveStatsTimer.Processed(); 71 CriticalSectionScoped cs(process_crit_sect_.get()); 72 if (_receiveStatsCallback != NULL) { 73 uint32_t bitRate; 74 uint32_t frameRate; 75 _receiver.ReceiveStatistics(&bitRate, &frameRate); 76 _receiveStatsCallback->OnReceiveRatesUpdated(bitRate, frameRate); 77 } 78 79 if (_decoderTimingCallback != NULL) { 80 int decode_ms; 81 int max_decode_ms; 82 int current_delay_ms; 83 int target_delay_ms; 84 int jitter_buffer_ms; 85 int min_playout_delay_ms; 86 int render_delay_ms; 87 _timing.GetTimings(&decode_ms, &max_decode_ms, ¤t_delay_ms, 88 &target_delay_ms, &jitter_buffer_ms, 89 &min_playout_delay_ms, &render_delay_ms); 90 _decoderTimingCallback->OnDecoderTiming( 91 decode_ms, max_decode_ms, current_delay_ms, target_delay_ms, 92 jitter_buffer_ms, min_playout_delay_ms, render_delay_ms); 93 } 94 95 // Size of render buffer. 96 if (render_buffer_callback_) { 97 int buffer_size_ms = _receiver.RenderBufferSizeMs(); 98 render_buffer_callback_->RenderBufferSizeMs(buffer_size_ms); 99 } 100 } 101 102 // Key frame requests 103 if (_keyRequestTimer.TimeUntilProcess() == 0) { 104 _keyRequestTimer.Processed(); 105 bool request_key_frame = false; 106 { 107 CriticalSectionScoped cs(process_crit_sect_.get()); 108 request_key_frame = _scheduleKeyRequest && _frameTypeCallback != NULL; 109 } 110 if (request_key_frame) { 111 const int32_t ret = RequestKeyFrame(); 112 if (ret != VCM_OK && returnValue == VCM_OK) { 113 returnValue = ret; 114 } 115 } 116 } 117 118 // Packet retransmission requests 119 // TODO(holmer): Add API for changing Process interval and make sure it's 120 // disabled when NACK is off. 121 if (_retransmissionTimer.TimeUntilProcess() == 0) { 122 _retransmissionTimer.Processed(); 123 bool callback_registered = false; 124 uint16_t length; 125 { 126 CriticalSectionScoped cs(process_crit_sect_.get()); 127 length = max_nack_list_size_; 128 callback_registered = _packetRequestCallback != NULL; 129 } 130 if (callback_registered && length > 0) { 131 // Collect sequence numbers from the default receiver. 132 bool request_key_frame = false; 133 std::vector<uint16_t> nackList = _receiver.NackList(&request_key_frame); 134 int32_t ret = VCM_OK; 135 if (request_key_frame) { 136 ret = RequestKeyFrame(); 137 if (ret != VCM_OK && returnValue == VCM_OK) { 138 returnValue = ret; 139 } 140 } 141 if (ret == VCM_OK && !nackList.empty()) { 142 CriticalSectionScoped cs(process_crit_sect_.get()); 143 if (_packetRequestCallback != NULL) { 144 _packetRequestCallback->ResendPackets(&nackList[0], nackList.size()); 145 } 146 } 147 } 148 } 149 150 return returnValue; 151} 152 153int64_t VideoReceiver::TimeUntilNextProcess() { 154 int64_t timeUntilNextProcess = _receiveStatsTimer.TimeUntilProcess(); 155 if (_receiver.NackMode() != kNoNack) { 156 // We need a Process call more often if we are relying on 157 // retransmissions 158 timeUntilNextProcess = 159 VCM_MIN(timeUntilNextProcess, _retransmissionTimer.TimeUntilProcess()); 160 } 161 timeUntilNextProcess = 162 VCM_MIN(timeUntilNextProcess, _keyRequestTimer.TimeUntilProcess()); 163 164 return timeUntilNextProcess; 165} 166 167int32_t VideoReceiver::SetReceiveChannelParameters(int64_t rtt) { 168 CriticalSectionScoped receiveCs(_receiveCritSect); 169 _receiver.UpdateRtt(rtt); 170 return 0; 171} 172 173// Enable or disable a video protection method. 174// Note: This API should be deprecated, as it does not offer a distinction 175// between the protection method and decoding with or without errors. If such a 176// behavior is desired, use the following API: SetReceiverRobustnessMode. 177int32_t VideoReceiver::SetVideoProtection(VCMVideoProtection videoProtection, 178 bool enable) { 179 // By default, do not decode with errors. 180 _receiver.SetDecodeErrorMode(kNoErrors); 181 switch (videoProtection) { 182 case kProtectionNack: { 183 RTC_DCHECK(enable); 184 _receiver.SetNackMode(kNack, -1, -1); 185 break; 186 } 187 188 case kProtectionNackFEC: { 189 CriticalSectionScoped cs(_receiveCritSect); 190 RTC_DCHECK(enable); 191 _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1); 192 _receiver.SetDecodeErrorMode(kNoErrors); 193 break; 194 } 195 case kProtectionFEC: 196 case kProtectionNone: 197 // No receiver-side protection. 198 RTC_DCHECK(enable); 199 _receiver.SetNackMode(kNoNack, -1, -1); 200 _receiver.SetDecodeErrorMode(kWithErrors); 201 break; 202 } 203 return VCM_OK; 204} 205 206// Register a receive callback. Will be called whenever there is a new frame 207// ready for rendering. 208int32_t VideoReceiver::RegisterReceiveCallback( 209 VCMReceiveCallback* receiveCallback) { 210 CriticalSectionScoped cs(_receiveCritSect); 211 _decodedFrameCallback.SetUserReceiveCallback(receiveCallback); 212 return VCM_OK; 213} 214 215int32_t VideoReceiver::RegisterReceiveStatisticsCallback( 216 VCMReceiveStatisticsCallback* receiveStats) { 217 CriticalSectionScoped cs(process_crit_sect_.get()); 218 _receiver.RegisterStatsCallback(receiveStats); 219 _receiveStatsCallback = receiveStats; 220 return VCM_OK; 221} 222 223int32_t VideoReceiver::RegisterDecoderTimingCallback( 224 VCMDecoderTimingCallback* decoderTiming) { 225 CriticalSectionScoped cs(process_crit_sect_.get()); 226 _decoderTimingCallback = decoderTiming; 227 return VCM_OK; 228} 229 230// Register an externally defined decoder object. 231void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder, 232 uint8_t payloadType) { 233 CriticalSectionScoped cs(_receiveCritSect); 234 if (externalDecoder == NULL) { 235 // Make sure the VCM updates the decoder next time it decodes. 236 _decoder = NULL; 237 RTC_CHECK(_codecDataBase.DeregisterExternalDecoder(payloadType)); 238 return; 239 } 240 _codecDataBase.RegisterExternalDecoder(externalDecoder, payloadType); 241} 242 243// Register a frame type request callback. 244int32_t VideoReceiver::RegisterFrameTypeCallback( 245 VCMFrameTypeCallback* frameTypeCallback) { 246 CriticalSectionScoped cs(process_crit_sect_.get()); 247 _frameTypeCallback = frameTypeCallback; 248 return VCM_OK; 249} 250 251int32_t VideoReceiver::RegisterPacketRequestCallback( 252 VCMPacketRequestCallback* callback) { 253 CriticalSectionScoped cs(process_crit_sect_.get()); 254 _packetRequestCallback = callback; 255 return VCM_OK; 256} 257 258int VideoReceiver::RegisterRenderBufferSizeCallback( 259 VCMRenderBufferSizeCallback* callback) { 260 CriticalSectionScoped cs(process_crit_sect_.get()); 261 render_buffer_callback_ = callback; 262 return VCM_OK; 263} 264 265void VideoReceiver::TriggerDecoderShutdown() { 266 _receiver.TriggerDecoderShutdown(); 267} 268 269// Decode next frame, blocking. 270// Should be called as often as possible to get the most out of the decoder. 271int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) { 272 int64_t nextRenderTimeMs; 273 bool prefer_late_decoding = false; 274 { 275 CriticalSectionScoped cs(_receiveCritSect); 276 prefer_late_decoding = _codecDataBase.PrefersLateDecoding(); 277 } 278 279 VCMEncodedFrame* frame = _receiver.FrameForDecoding( 280 maxWaitTimeMs, &nextRenderTimeMs, prefer_late_decoding); 281 282 if (!frame) 283 return VCM_FRAME_NOT_READY; 284 285 CriticalSectionScoped cs(_receiveCritSect); 286 287 // If this frame was too late, we should adjust the delay accordingly 288 _timing.UpdateCurrentDelay(frame->RenderTimeMs(), 289 clock_->TimeInMilliseconds()); 290 291 if (pre_decode_image_callback_) { 292 EncodedImage encoded_image(frame->EncodedImage()); 293 int qp = -1; 294 if (qp_parser_.GetQp(*frame, &qp)) { 295 encoded_image.qp_ = qp; 296 } 297 pre_decode_image_callback_->Encoded(encoded_image, frame->CodecSpecific(), 298 NULL); 299 } 300 301#ifdef DEBUG_DECODER_BIT_STREAM 302 if (_bitStreamBeforeDecoder != NULL) { 303 // Write bit stream to file for debugging purposes 304 if (fwrite(frame->Buffer(), 1, frame->Length(), _bitStreamBeforeDecoder) != 305 frame->Length()) { 306 return -1; 307 } 308 } 309#endif 310 const int32_t ret = Decode(*frame); 311 _receiver.ReleaseFrame(frame); 312 return ret; 313} 314 315int32_t VideoReceiver::RequestSliceLossIndication( 316 const uint64_t pictureID) const { 317 TRACE_EVENT1("webrtc", "RequestSLI", "picture_id", pictureID); 318 CriticalSectionScoped cs(process_crit_sect_.get()); 319 if (_frameTypeCallback != NULL) { 320 const int32_t ret = 321 _frameTypeCallback->SliceLossIndicationRequest(pictureID); 322 if (ret < 0) { 323 return ret; 324 } 325 } else { 326 return VCM_MISSING_CALLBACK; 327 } 328 return VCM_OK; 329} 330 331int32_t VideoReceiver::RequestKeyFrame() { 332 TRACE_EVENT0("webrtc", "RequestKeyFrame"); 333 CriticalSectionScoped process_cs(process_crit_sect_.get()); 334 if (_frameTypeCallback != NULL) { 335 const int32_t ret = _frameTypeCallback->RequestKeyFrame(); 336 if (ret < 0) { 337 return ret; 338 } 339 _scheduleKeyRequest = false; 340 } else { 341 return VCM_MISSING_CALLBACK; 342 } 343 return VCM_OK; 344} 345 346// Must be called from inside the receive side critical section. 347int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) { 348 TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame.TimeStamp(), "Decode", 349 "type", frame.FrameType()); 350 // Change decoder if payload type has changed 351 _decoder = _codecDataBase.GetDecoder(frame, &_decodedFrameCallback); 352 if (_decoder == NULL) { 353 return VCM_NO_CODEC_REGISTERED; 354 } 355 // Decode a frame 356 int32_t ret = _decoder->Decode(frame, clock_->TimeInMilliseconds()); 357 358 // Check for failed decoding, run frame type request callback if needed. 359 bool request_key_frame = false; 360 if (ret < 0) { 361 if (ret == VCM_ERROR_REQUEST_SLI) { 362 return RequestSliceLossIndication( 363 _decodedFrameCallback.LastReceivedPictureID() + 1); 364 } else { 365 request_key_frame = true; 366 } 367 } else if (ret == VCM_REQUEST_SLI) { 368 ret = RequestSliceLossIndication( 369 _decodedFrameCallback.LastReceivedPictureID() + 1); 370 } 371 if (!frame.Complete() || frame.MissingFrame()) { 372 request_key_frame = true; 373 ret = VCM_OK; 374 } 375 if (request_key_frame) { 376 CriticalSectionScoped cs(process_crit_sect_.get()); 377 _scheduleKeyRequest = true; 378 } 379 TRACE_EVENT_ASYNC_END0("webrtc", "Video", frame.TimeStamp()); 380 return ret; 381} 382 383// Reset the decoder state 384int32_t VideoReceiver::ResetDecoder() { 385 bool reset_key_request = false; 386 { 387 CriticalSectionScoped cs(_receiveCritSect); 388 if (_decoder != NULL) { 389 _receiver.Reset(); 390 _timing.Reset(); 391 reset_key_request = true; 392 _decoder->Reset(); 393 } 394 } 395 if (reset_key_request) { 396 CriticalSectionScoped cs(process_crit_sect_.get()); 397 _scheduleKeyRequest = false; 398 } 399 return VCM_OK; 400} 401 402// Register possible receive codecs, can be called multiple times 403int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec, 404 int32_t numberOfCores, 405 bool requireKeyFrame) { 406 CriticalSectionScoped cs(_receiveCritSect); 407 if (receiveCodec == NULL) { 408 return VCM_PARAMETER_ERROR; 409 } 410 if (!_codecDataBase.RegisterReceiveCodec(receiveCodec, numberOfCores, 411 requireKeyFrame)) { 412 return -1; 413 } 414 return 0; 415} 416 417// Get current received codec 418int32_t VideoReceiver::ReceiveCodec(VideoCodec* currentReceiveCodec) const { 419 CriticalSectionScoped cs(_receiveCritSect); 420 if (currentReceiveCodec == NULL) { 421 return VCM_PARAMETER_ERROR; 422 } 423 return _codecDataBase.ReceiveCodec(currentReceiveCodec) ? 0 : -1; 424} 425 426// Get current received codec 427VideoCodecType VideoReceiver::ReceiveCodec() const { 428 CriticalSectionScoped cs(_receiveCritSect); 429 return _codecDataBase.ReceiveCodec(); 430} 431 432// Incoming packet from network parsed and ready for decode, non blocking. 433int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload, 434 size_t payloadLength, 435 const WebRtcRTPHeader& rtpInfo) { 436 if (rtpInfo.frameType == kVideoFrameKey) { 437 TRACE_EVENT1("webrtc", "VCM::PacketKeyFrame", "seqnum", 438 rtpInfo.header.sequenceNumber); 439 } 440 if (incomingPayload == NULL) { 441 // The jitter buffer doesn't handle non-zero payload lengths for packets 442 // without payload. 443 // TODO(holmer): We should fix this in the jitter buffer. 444 payloadLength = 0; 445 } 446 const VCMPacket packet(incomingPayload, payloadLength, rtpInfo); 447 int32_t ret = _receiver.InsertPacket(packet, rtpInfo.type.Video.width, 448 rtpInfo.type.Video.height); 449 // TODO(holmer): Investigate if this somehow should use the key frame 450 // request scheduling to throttle the requests. 451 if (ret == VCM_FLUSH_INDICATOR) { 452 RequestKeyFrame(); 453 ResetDecoder(); 454 } else if (ret < 0) { 455 return ret; 456 } 457 return VCM_OK; 458} 459 460// Minimum playout delay (used for lip-sync). This is the minimum delay required 461// to sync with audio. Not included in VideoCodingModule::Delay() 462// Defaults to 0 ms. 463int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) { 464 _timing.set_min_playout_delay(minPlayoutDelayMs); 465 return VCM_OK; 466} 467 468// The estimated delay caused by rendering, defaults to 469// kDefaultRenderDelayMs = 10 ms 470int32_t VideoReceiver::SetRenderDelay(uint32_t timeMS) { 471 _timing.set_render_delay(timeMS); 472 return VCM_OK; 473} 474 475// Current video delay 476int32_t VideoReceiver::Delay() const { 477 return _timing.TargetVideoDelay(); 478} 479 480uint32_t VideoReceiver::DiscardedPackets() const { 481 return _receiver.DiscardedPackets(); 482} 483 484int VideoReceiver::SetReceiverRobustnessMode( 485 ReceiverRobustness robustnessMode, 486 VCMDecodeErrorMode decode_error_mode) { 487 CriticalSectionScoped cs(_receiveCritSect); 488 switch (robustnessMode) { 489 case VideoCodingModule::kNone: 490 _receiver.SetNackMode(kNoNack, -1, -1); 491 break; 492 case VideoCodingModule::kHardNack: 493 // Always wait for retransmissions (except when decoding with errors). 494 _receiver.SetNackMode(kNack, -1, -1); 495 break; 496 case VideoCodingModule::kSoftNack: 497#if 1 498 assert(false); // TODO(hlundin): Not completed. 499 return VCM_NOT_IMPLEMENTED; 500#else 501 // Enable hybrid NACK/FEC. Always wait for retransmissions and don't add 502 // extra delay when RTT is above kLowRttNackMs. 503 _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1); 504 break; 505#endif 506 case VideoCodingModule::kReferenceSelection: 507#if 1 508 assert(false); // TODO(hlundin): Not completed. 509 return VCM_NOT_IMPLEMENTED; 510#else 511 if (decode_error_mode == kNoErrors) { 512 return VCM_PARAMETER_ERROR; 513 } 514 _receiver.SetNackMode(kNoNack, -1, -1); 515 break; 516#endif 517 } 518 _receiver.SetDecodeErrorMode(decode_error_mode); 519 return VCM_OK; 520} 521 522void VideoReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) { 523 CriticalSectionScoped cs(_receiveCritSect); 524 _receiver.SetDecodeErrorMode(decode_error_mode); 525} 526 527void VideoReceiver::SetNackSettings(size_t max_nack_list_size, 528 int max_packet_age_to_nack, 529 int max_incomplete_time_ms) { 530 if (max_nack_list_size != 0) { 531 CriticalSectionScoped process_cs(process_crit_sect_.get()); 532 max_nack_list_size_ = max_nack_list_size; 533 } 534 _receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack, 535 max_incomplete_time_ms); 536} 537 538int VideoReceiver::SetMinReceiverDelay(int desired_delay_ms) { 539 return _receiver.SetMinReceiverDelay(desired_delay_ms); 540} 541 542void VideoReceiver::RegisterPreDecodeImageCallback( 543 EncodedImageCallback* observer) { 544 CriticalSectionScoped cs(_receiveCritSect); 545 pre_decode_image_callback_ = observer; 546} 547 548} // namespace vcm 549} // namespace webrtc 550