1/*
2 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/base/checks.h"
12#include "webrtc/base/logging.h"
13#include "webrtc/base/trace_event.h"
14#include "webrtc/common_types.h"
15#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
16#include "webrtc/modules/video_coding/include/video_codec_interface.h"
17#include "webrtc/modules/video_coding/encoded_frame.h"
18#include "webrtc/modules/video_coding/jitter_buffer.h"
19#include "webrtc/modules/video_coding/packet.h"
20#include "webrtc/modules/video_coding/video_coding_impl.h"
21#include "webrtc/system_wrappers/include/clock.h"
22
23// #define DEBUG_DECODER_BIT_STREAM
24
25namespace webrtc {
26namespace vcm {
27
28VideoReceiver::VideoReceiver(Clock* clock, EventFactory* event_factory)
29    : clock_(clock),
30      process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
31      _receiveCritSect(CriticalSectionWrapper::CreateCriticalSection()),
32      _timing(clock_),
33      _receiver(&_timing, clock_, event_factory),
34      _decodedFrameCallback(&_timing, clock_),
35      _frameTypeCallback(NULL),
36      _receiveStatsCallback(NULL),
37      _decoderTimingCallback(NULL),
38      _packetRequestCallback(NULL),
39      render_buffer_callback_(NULL),
40      _decoder(NULL),
41#ifdef DEBUG_DECODER_BIT_STREAM
42      _bitStreamBeforeDecoder(NULL),
43#endif
44      _frameFromFile(),
45      _scheduleKeyRequest(false),
46      max_nack_list_size_(0),
47      pre_decode_image_callback_(NULL),
48      _codecDataBase(nullptr, nullptr),
49      _receiveStatsTimer(1000, clock_),
50      _retransmissionTimer(10, clock_),
51      _keyRequestTimer(500, clock_) {
52  assert(clock_);
53#ifdef DEBUG_DECODER_BIT_STREAM
54  _bitStreamBeforeDecoder = fopen("decoderBitStream.bit", "wb");
55#endif
56}
57
58VideoReceiver::~VideoReceiver() {
59  delete _receiveCritSect;
60#ifdef DEBUG_DECODER_BIT_STREAM
61  fclose(_bitStreamBeforeDecoder);
62#endif
63}
64
65int32_t VideoReceiver::Process() {
66  int32_t returnValue = VCM_OK;
67
68  // Receive-side statistics
69  if (_receiveStatsTimer.TimeUntilProcess() == 0) {
70    _receiveStatsTimer.Processed();
71    CriticalSectionScoped cs(process_crit_sect_.get());
72    if (_receiveStatsCallback != NULL) {
73      uint32_t bitRate;
74      uint32_t frameRate;
75      _receiver.ReceiveStatistics(&bitRate, &frameRate);
76      _receiveStatsCallback->OnReceiveRatesUpdated(bitRate, frameRate);
77    }
78
79    if (_decoderTimingCallback != NULL) {
80      int decode_ms;
81      int max_decode_ms;
82      int current_delay_ms;
83      int target_delay_ms;
84      int jitter_buffer_ms;
85      int min_playout_delay_ms;
86      int render_delay_ms;
87      _timing.GetTimings(&decode_ms, &max_decode_ms, &current_delay_ms,
88                         &target_delay_ms, &jitter_buffer_ms,
89                         &min_playout_delay_ms, &render_delay_ms);
90      _decoderTimingCallback->OnDecoderTiming(
91          decode_ms, max_decode_ms, current_delay_ms, target_delay_ms,
92          jitter_buffer_ms, min_playout_delay_ms, render_delay_ms);
93    }
94
95    // Size of render buffer.
96    if (render_buffer_callback_) {
97      int buffer_size_ms = _receiver.RenderBufferSizeMs();
98      render_buffer_callback_->RenderBufferSizeMs(buffer_size_ms);
99    }
100  }
101
102  // Key frame requests
103  if (_keyRequestTimer.TimeUntilProcess() == 0) {
104    _keyRequestTimer.Processed();
105    bool request_key_frame = false;
106    {
107      CriticalSectionScoped cs(process_crit_sect_.get());
108      request_key_frame = _scheduleKeyRequest && _frameTypeCallback != NULL;
109    }
110    if (request_key_frame) {
111      const int32_t ret = RequestKeyFrame();
112      if (ret != VCM_OK && returnValue == VCM_OK) {
113        returnValue = ret;
114      }
115    }
116  }
117
118  // Packet retransmission requests
119  // TODO(holmer): Add API for changing Process interval and make sure it's
120  // disabled when NACK is off.
121  if (_retransmissionTimer.TimeUntilProcess() == 0) {
122    _retransmissionTimer.Processed();
123    bool callback_registered = false;
124    uint16_t length;
125    {
126      CriticalSectionScoped cs(process_crit_sect_.get());
127      length = max_nack_list_size_;
128      callback_registered = _packetRequestCallback != NULL;
129    }
130    if (callback_registered && length > 0) {
131      // Collect sequence numbers from the default receiver.
132      bool request_key_frame = false;
133      std::vector<uint16_t> nackList = _receiver.NackList(&request_key_frame);
134      int32_t ret = VCM_OK;
135      if (request_key_frame) {
136        ret = RequestKeyFrame();
137        if (ret != VCM_OK && returnValue == VCM_OK) {
138          returnValue = ret;
139        }
140      }
141      if (ret == VCM_OK && !nackList.empty()) {
142        CriticalSectionScoped cs(process_crit_sect_.get());
143        if (_packetRequestCallback != NULL) {
144          _packetRequestCallback->ResendPackets(&nackList[0], nackList.size());
145        }
146      }
147    }
148  }
149
150  return returnValue;
151}
152
153int64_t VideoReceiver::TimeUntilNextProcess() {
154  int64_t timeUntilNextProcess = _receiveStatsTimer.TimeUntilProcess();
155  if (_receiver.NackMode() != kNoNack) {
156    // We need a Process call more often if we are relying on
157    // retransmissions
158    timeUntilNextProcess =
159        VCM_MIN(timeUntilNextProcess, _retransmissionTimer.TimeUntilProcess());
160  }
161  timeUntilNextProcess =
162      VCM_MIN(timeUntilNextProcess, _keyRequestTimer.TimeUntilProcess());
163
164  return timeUntilNextProcess;
165}
166
167int32_t VideoReceiver::SetReceiveChannelParameters(int64_t rtt) {
168  CriticalSectionScoped receiveCs(_receiveCritSect);
169  _receiver.UpdateRtt(rtt);
170  return 0;
171}
172
173// Enable or disable a video protection method.
174// Note: This API should be deprecated, as it does not offer a distinction
175// between the protection method and decoding with or without errors. If such a
176// behavior is desired, use the following API: SetReceiverRobustnessMode.
177int32_t VideoReceiver::SetVideoProtection(VCMVideoProtection videoProtection,
178                                          bool enable) {
179  // By default, do not decode with errors.
180  _receiver.SetDecodeErrorMode(kNoErrors);
181  switch (videoProtection) {
182    case kProtectionNack: {
183      RTC_DCHECK(enable);
184      _receiver.SetNackMode(kNack, -1, -1);
185      break;
186    }
187
188    case kProtectionNackFEC: {
189      CriticalSectionScoped cs(_receiveCritSect);
190      RTC_DCHECK(enable);
191      _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1);
192      _receiver.SetDecodeErrorMode(kNoErrors);
193      break;
194    }
195    case kProtectionFEC:
196    case kProtectionNone:
197      // No receiver-side protection.
198      RTC_DCHECK(enable);
199      _receiver.SetNackMode(kNoNack, -1, -1);
200      _receiver.SetDecodeErrorMode(kWithErrors);
201      break;
202  }
203  return VCM_OK;
204}
205
206// Register a receive callback. Will be called whenever there is a new frame
207// ready for rendering.
208int32_t VideoReceiver::RegisterReceiveCallback(
209    VCMReceiveCallback* receiveCallback) {
210  CriticalSectionScoped cs(_receiveCritSect);
211  _decodedFrameCallback.SetUserReceiveCallback(receiveCallback);
212  return VCM_OK;
213}
214
215int32_t VideoReceiver::RegisterReceiveStatisticsCallback(
216    VCMReceiveStatisticsCallback* receiveStats) {
217  CriticalSectionScoped cs(process_crit_sect_.get());
218  _receiver.RegisterStatsCallback(receiveStats);
219  _receiveStatsCallback = receiveStats;
220  return VCM_OK;
221}
222
223int32_t VideoReceiver::RegisterDecoderTimingCallback(
224    VCMDecoderTimingCallback* decoderTiming) {
225  CriticalSectionScoped cs(process_crit_sect_.get());
226  _decoderTimingCallback = decoderTiming;
227  return VCM_OK;
228}
229
230// Register an externally defined decoder object.
231void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder,
232                                            uint8_t payloadType) {
233  CriticalSectionScoped cs(_receiveCritSect);
234  if (externalDecoder == NULL) {
235    // Make sure the VCM updates the decoder next time it decodes.
236    _decoder = NULL;
237    RTC_CHECK(_codecDataBase.DeregisterExternalDecoder(payloadType));
238    return;
239  }
240  _codecDataBase.RegisterExternalDecoder(externalDecoder, payloadType);
241}
242
243// Register a frame type request callback.
244int32_t VideoReceiver::RegisterFrameTypeCallback(
245    VCMFrameTypeCallback* frameTypeCallback) {
246  CriticalSectionScoped cs(process_crit_sect_.get());
247  _frameTypeCallback = frameTypeCallback;
248  return VCM_OK;
249}
250
251int32_t VideoReceiver::RegisterPacketRequestCallback(
252    VCMPacketRequestCallback* callback) {
253  CriticalSectionScoped cs(process_crit_sect_.get());
254  _packetRequestCallback = callback;
255  return VCM_OK;
256}
257
258int VideoReceiver::RegisterRenderBufferSizeCallback(
259    VCMRenderBufferSizeCallback* callback) {
260  CriticalSectionScoped cs(process_crit_sect_.get());
261  render_buffer_callback_ = callback;
262  return VCM_OK;
263}
264
265void VideoReceiver::TriggerDecoderShutdown() {
266  _receiver.TriggerDecoderShutdown();
267}
268
269// Decode next frame, blocking.
270// Should be called as often as possible to get the most out of the decoder.
271int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) {
272  int64_t nextRenderTimeMs;
273  bool prefer_late_decoding = false;
274  {
275    CriticalSectionScoped cs(_receiveCritSect);
276    prefer_late_decoding = _codecDataBase.PrefersLateDecoding();
277  }
278
279  VCMEncodedFrame* frame = _receiver.FrameForDecoding(
280      maxWaitTimeMs, &nextRenderTimeMs, prefer_late_decoding);
281
282  if (!frame)
283    return VCM_FRAME_NOT_READY;
284
285  CriticalSectionScoped cs(_receiveCritSect);
286
287  // If this frame was too late, we should adjust the delay accordingly
288  _timing.UpdateCurrentDelay(frame->RenderTimeMs(),
289                             clock_->TimeInMilliseconds());
290
291  if (pre_decode_image_callback_) {
292    EncodedImage encoded_image(frame->EncodedImage());
293    int qp = -1;
294    if (qp_parser_.GetQp(*frame, &qp)) {
295      encoded_image.qp_ = qp;
296    }
297    pre_decode_image_callback_->Encoded(encoded_image, frame->CodecSpecific(),
298                                        NULL);
299  }
300
301#ifdef DEBUG_DECODER_BIT_STREAM
302  if (_bitStreamBeforeDecoder != NULL) {
303    // Write bit stream to file for debugging purposes
304    if (fwrite(frame->Buffer(), 1, frame->Length(), _bitStreamBeforeDecoder) !=
305        frame->Length()) {
306      return -1;
307    }
308  }
309#endif
310  const int32_t ret = Decode(*frame);
311  _receiver.ReleaseFrame(frame);
312  return ret;
313}
314
315int32_t VideoReceiver::RequestSliceLossIndication(
316    const uint64_t pictureID) const {
317  TRACE_EVENT1("webrtc", "RequestSLI", "picture_id", pictureID);
318  CriticalSectionScoped cs(process_crit_sect_.get());
319  if (_frameTypeCallback != NULL) {
320    const int32_t ret =
321        _frameTypeCallback->SliceLossIndicationRequest(pictureID);
322    if (ret < 0) {
323      return ret;
324    }
325  } else {
326    return VCM_MISSING_CALLBACK;
327  }
328  return VCM_OK;
329}
330
331int32_t VideoReceiver::RequestKeyFrame() {
332  TRACE_EVENT0("webrtc", "RequestKeyFrame");
333  CriticalSectionScoped process_cs(process_crit_sect_.get());
334  if (_frameTypeCallback != NULL) {
335    const int32_t ret = _frameTypeCallback->RequestKeyFrame();
336    if (ret < 0) {
337      return ret;
338    }
339    _scheduleKeyRequest = false;
340  } else {
341    return VCM_MISSING_CALLBACK;
342  }
343  return VCM_OK;
344}
345
346// Must be called from inside the receive side critical section.
347int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) {
348  TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame.TimeStamp(), "Decode",
349                          "type", frame.FrameType());
350  // Change decoder if payload type has changed
351  _decoder = _codecDataBase.GetDecoder(frame, &_decodedFrameCallback);
352  if (_decoder == NULL) {
353    return VCM_NO_CODEC_REGISTERED;
354  }
355  // Decode a frame
356  int32_t ret = _decoder->Decode(frame, clock_->TimeInMilliseconds());
357
358  // Check for failed decoding, run frame type request callback if needed.
359  bool request_key_frame = false;
360  if (ret < 0) {
361    if (ret == VCM_ERROR_REQUEST_SLI) {
362      return RequestSliceLossIndication(
363          _decodedFrameCallback.LastReceivedPictureID() + 1);
364    } else {
365      request_key_frame = true;
366    }
367  } else if (ret == VCM_REQUEST_SLI) {
368    ret = RequestSliceLossIndication(
369        _decodedFrameCallback.LastReceivedPictureID() + 1);
370  }
371  if (!frame.Complete() || frame.MissingFrame()) {
372    request_key_frame = true;
373    ret = VCM_OK;
374  }
375  if (request_key_frame) {
376    CriticalSectionScoped cs(process_crit_sect_.get());
377    _scheduleKeyRequest = true;
378  }
379  TRACE_EVENT_ASYNC_END0("webrtc", "Video", frame.TimeStamp());
380  return ret;
381}
382
383// Reset the decoder state
384int32_t VideoReceiver::ResetDecoder() {
385  bool reset_key_request = false;
386  {
387    CriticalSectionScoped cs(_receiveCritSect);
388    if (_decoder != NULL) {
389      _receiver.Reset();
390      _timing.Reset();
391      reset_key_request = true;
392      _decoder->Reset();
393    }
394  }
395  if (reset_key_request) {
396    CriticalSectionScoped cs(process_crit_sect_.get());
397    _scheduleKeyRequest = false;
398  }
399  return VCM_OK;
400}
401
402// Register possible receive codecs, can be called multiple times
403int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec,
404                                            int32_t numberOfCores,
405                                            bool requireKeyFrame) {
406  CriticalSectionScoped cs(_receiveCritSect);
407  if (receiveCodec == NULL) {
408    return VCM_PARAMETER_ERROR;
409  }
410  if (!_codecDataBase.RegisterReceiveCodec(receiveCodec, numberOfCores,
411                                           requireKeyFrame)) {
412    return -1;
413  }
414  return 0;
415}
416
417// Get current received codec
418int32_t VideoReceiver::ReceiveCodec(VideoCodec* currentReceiveCodec) const {
419  CriticalSectionScoped cs(_receiveCritSect);
420  if (currentReceiveCodec == NULL) {
421    return VCM_PARAMETER_ERROR;
422  }
423  return _codecDataBase.ReceiveCodec(currentReceiveCodec) ? 0 : -1;
424}
425
426// Get current received codec
427VideoCodecType VideoReceiver::ReceiveCodec() const {
428  CriticalSectionScoped cs(_receiveCritSect);
429  return _codecDataBase.ReceiveCodec();
430}
431
432// Incoming packet from network parsed and ready for decode, non blocking.
433int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload,
434                                      size_t payloadLength,
435                                      const WebRtcRTPHeader& rtpInfo) {
436  if (rtpInfo.frameType == kVideoFrameKey) {
437    TRACE_EVENT1("webrtc", "VCM::PacketKeyFrame", "seqnum",
438                 rtpInfo.header.sequenceNumber);
439  }
440  if (incomingPayload == NULL) {
441    // The jitter buffer doesn't handle non-zero payload lengths for packets
442    // without payload.
443    // TODO(holmer): We should fix this in the jitter buffer.
444    payloadLength = 0;
445  }
446  const VCMPacket packet(incomingPayload, payloadLength, rtpInfo);
447  int32_t ret = _receiver.InsertPacket(packet, rtpInfo.type.Video.width,
448                                       rtpInfo.type.Video.height);
449  // TODO(holmer): Investigate if this somehow should use the key frame
450  // request scheduling to throttle the requests.
451  if (ret == VCM_FLUSH_INDICATOR) {
452    RequestKeyFrame();
453    ResetDecoder();
454  } else if (ret < 0) {
455    return ret;
456  }
457  return VCM_OK;
458}
459
460// Minimum playout delay (used for lip-sync). This is the minimum delay required
461// to sync with audio. Not included in  VideoCodingModule::Delay()
462// Defaults to 0 ms.
463int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) {
464  _timing.set_min_playout_delay(minPlayoutDelayMs);
465  return VCM_OK;
466}
467
468// The estimated delay caused by rendering, defaults to
469// kDefaultRenderDelayMs = 10 ms
470int32_t VideoReceiver::SetRenderDelay(uint32_t timeMS) {
471  _timing.set_render_delay(timeMS);
472  return VCM_OK;
473}
474
475// Current video delay
476int32_t VideoReceiver::Delay() const {
477  return _timing.TargetVideoDelay();
478}
479
480uint32_t VideoReceiver::DiscardedPackets() const {
481  return _receiver.DiscardedPackets();
482}
483
484int VideoReceiver::SetReceiverRobustnessMode(
485    ReceiverRobustness robustnessMode,
486    VCMDecodeErrorMode decode_error_mode) {
487  CriticalSectionScoped cs(_receiveCritSect);
488  switch (robustnessMode) {
489    case VideoCodingModule::kNone:
490      _receiver.SetNackMode(kNoNack, -1, -1);
491      break;
492    case VideoCodingModule::kHardNack:
493      // Always wait for retransmissions (except when decoding with errors).
494      _receiver.SetNackMode(kNack, -1, -1);
495      break;
496    case VideoCodingModule::kSoftNack:
497#if 1
498      assert(false);  // TODO(hlundin): Not completed.
499      return VCM_NOT_IMPLEMENTED;
500#else
501      // Enable hybrid NACK/FEC. Always wait for retransmissions and don't add
502      // extra delay when RTT is above kLowRttNackMs.
503      _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1);
504      break;
505#endif
506    case VideoCodingModule::kReferenceSelection:
507#if 1
508      assert(false);  // TODO(hlundin): Not completed.
509      return VCM_NOT_IMPLEMENTED;
510#else
511      if (decode_error_mode == kNoErrors) {
512        return VCM_PARAMETER_ERROR;
513      }
514      _receiver.SetNackMode(kNoNack, -1, -1);
515      break;
516#endif
517  }
518  _receiver.SetDecodeErrorMode(decode_error_mode);
519  return VCM_OK;
520}
521
522void VideoReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
523  CriticalSectionScoped cs(_receiveCritSect);
524  _receiver.SetDecodeErrorMode(decode_error_mode);
525}
526
527void VideoReceiver::SetNackSettings(size_t max_nack_list_size,
528                                    int max_packet_age_to_nack,
529                                    int max_incomplete_time_ms) {
530  if (max_nack_list_size != 0) {
531    CriticalSectionScoped process_cs(process_crit_sect_.get());
532    max_nack_list_size_ = max_nack_list_size;
533  }
534  _receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack,
535                            max_incomplete_time_ms);
536}
537
538int VideoReceiver::SetMinReceiverDelay(int desired_delay_ms) {
539  return _receiver.SetMinReceiverDelay(desired_delay_ms);
540}
541
542void VideoReceiver::RegisterPreDecodeImageCallback(
543    EncodedImageCallback* observer) {
544  CriticalSectionScoped cs(_receiveCritSect);
545  pre_decode_image_callback_ = observer;
546}
547
548}  // namespace vcm
549}  // namespace webrtc
550