1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/modules/video_coding/main/source/receiver.h"
12
13#include <assert.h>
14
15#include <cstdlib>
16
17#include "webrtc/modules/video_coding/main/source/encoded_frame.h"
18#include "webrtc/modules/video_coding/main/source/internal_defines.h"
19#include "webrtc/modules/video_coding/main/source/media_opt_util.h"
20#include "webrtc/system_wrappers/interface/clock.h"
21#include "webrtc/system_wrappers/interface/logging.h"
22#include "webrtc/system_wrappers/interface/trace_event.h"
23
24namespace webrtc {
25
26enum { kMaxReceiverDelayMs = 10000 };
27
28VCMReceiver::VCMReceiver(VCMTiming* timing,
29                         Clock* clock,
30                         EventFactory* event_factory,
31                         bool master)
32    : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
33      clock_(clock),
34      master_(master),
35      jitter_buffer_(clock_, event_factory),
36      timing_(timing),
37      render_wait_event_(event_factory->CreateEvent()),
38      state_(kPassive),
39      max_video_delay_ms_(kMaxVideoDelayMs) {}
40
41VCMReceiver::~VCMReceiver() {
42  render_wait_event_->Set();
43  delete crit_sect_;
44}
45
46void VCMReceiver::Reset() {
47  CriticalSectionScoped cs(crit_sect_);
48  if (!jitter_buffer_.Running()) {
49    jitter_buffer_.Start();
50  } else {
51    jitter_buffer_.Flush();
52  }
53  render_wait_event_->Reset();
54  if (master_) {
55    state_ = kReceiving;
56  } else {
57    state_ = kPassive;
58  }
59}
60
61int32_t VCMReceiver::Initialize() {
62  Reset();
63  CriticalSectionScoped cs(crit_sect_);
64  if (!master_) {
65    SetNackMode(kNoNack, -1, -1);
66  }
67  return VCM_OK;
68}
69
70void VCMReceiver::UpdateRtt(uint32_t rtt) {
71  jitter_buffer_.UpdateRtt(rtt);
72}
73
74int32_t VCMReceiver::InsertPacket(const VCMPacket& packet,
75                                  uint16_t frame_width,
76                                  uint16_t frame_height) {
77  // Insert the packet into the jitter buffer. The packet can either be empty or
78  // contain media at this point.
79  bool retransmitted = false;
80  const VCMFrameBufferEnum ret = jitter_buffer_.InsertPacket(packet,
81                                                             &retransmitted);
82  if (ret == kOldPacket) {
83    return VCM_OK;
84  } else if (ret == kFlushIndicator) {
85    return VCM_FLUSH_INDICATOR;
86  } else if (ret < 0) {
87    return VCM_JITTER_BUFFER_ERROR;
88  }
89  if (ret == kCompleteSession && !retransmitted) {
90    // We don't want to include timestamps which have suffered from
91    // retransmission here, since we compensate with extra retransmission
92    // delay within the jitter estimate.
93    timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds());
94  }
95  return VCM_OK;
96}
97
98VCMEncodedFrame* VCMReceiver::FrameForDecoding(
99    uint16_t max_wait_time_ms,
100    int64_t& next_render_time_ms,
101    bool render_timing,
102    VCMReceiver* dual_receiver) {
103  const int64_t start_time_ms = clock_->TimeInMilliseconds();
104  uint32_t frame_timestamp = 0;
105  // Exhaust wait time to get a complete frame for decoding.
106  bool found_frame = jitter_buffer_.NextCompleteTimestamp(
107      max_wait_time_ms, &frame_timestamp);
108
109  if (!found_frame) {
110    // Get an incomplete frame when enabled.
111    const bool dual_receiver_enabled_and_passive = (dual_receiver != NULL &&
112        dual_receiver->State() == kPassive &&
113        dual_receiver->NackMode() == kNack);
114    if (dual_receiver_enabled_and_passive &&
115        !jitter_buffer_.CompleteSequenceWithNextFrame()) {
116      // Jitter buffer state might get corrupt with this frame.
117      dual_receiver->CopyJitterBufferStateFromReceiver(*this);
118    }
119    found_frame = jitter_buffer_.NextMaybeIncompleteTimestamp(
120        &frame_timestamp);
121  }
122
123  if (!found_frame) {
124    return NULL;
125  }
126
127  // We have a frame - Set timing and render timestamp.
128  timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs());
129  const int64_t now_ms = clock_->TimeInMilliseconds();
130  timing_->UpdateCurrentDelay(frame_timestamp);
131  next_render_time_ms = timing_->RenderTimeMs(frame_timestamp, now_ms);
132  // Check render timing.
133  bool timing_error = false;
134  // Assume that render timing errors are due to changes in the video stream.
135  if (next_render_time_ms < 0) {
136    timing_error = true;
137  } else if (std::abs(next_render_time_ms - now_ms) > max_video_delay_ms_) {
138    int frame_delay = static_cast<int>(std::abs(next_render_time_ms - now_ms));
139    LOG(LS_WARNING) << "A frame about to be decoded is out of the configured "
140                    << "delay bounds (" << frame_delay << " > "
141                    << max_video_delay_ms_
142                    << "). Resetting the video jitter buffer.";
143    timing_error = true;
144  } else if (static_cast<int>(timing_->TargetVideoDelay()) >
145             max_video_delay_ms_) {
146    LOG(LS_WARNING) << "The video target delay has grown larger than "
147                    << max_video_delay_ms_ << " ms. Resetting jitter buffer.";
148    timing_error = true;
149  }
150
151  if (timing_error) {
152    // Timing error => reset timing and flush the jitter buffer.
153    jitter_buffer_.Flush();
154    timing_->Reset();
155    return NULL;
156  }
157
158  if (!render_timing) {
159    // Decode frame as close as possible to the render timestamp.
160    const int32_t available_wait_time = max_wait_time_ms -
161        static_cast<int32_t>(clock_->TimeInMilliseconds() - start_time_ms);
162    uint16_t new_max_wait_time = static_cast<uint16_t>(
163        VCM_MAX(available_wait_time, 0));
164    uint32_t wait_time_ms = timing_->MaxWaitingTime(
165        next_render_time_ms, clock_->TimeInMilliseconds());
166    if (new_max_wait_time < wait_time_ms) {
167      // We're not allowed to wait until the frame is supposed to be rendered,
168      // waiting as long as we're allowed to avoid busy looping, and then return
169      // NULL. Next call to this function might return the frame.
170      render_wait_event_->Wait(max_wait_time_ms);
171      return NULL;
172    }
173    // Wait until it's time to render.
174    render_wait_event_->Wait(wait_time_ms);
175  }
176
177  // Extract the frame from the jitter buffer and set the render time.
178  VCMEncodedFrame* frame = jitter_buffer_.ExtractAndSetDecode(frame_timestamp);
179  if (frame == NULL) {
180    return NULL;
181  }
182  frame->SetRenderTime(next_render_time_ms);
183  TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame->TimeStamp(),
184                          "SetRenderTS", "render_time", next_render_time_ms);
185  if (dual_receiver != NULL) {
186    dual_receiver->UpdateState(*frame);
187  }
188  if (!frame->Complete()) {
189    // Update stats for incomplete frames.
190    bool retransmitted = false;
191    const int64_t last_packet_time_ms =
192        jitter_buffer_.LastPacketTime(frame, &retransmitted);
193    if (last_packet_time_ms >= 0 && !retransmitted) {
194      // We don't want to include timestamps which have suffered from
195      // retransmission here, since we compensate with extra retransmission
196      // delay within the jitter estimate.
197      timing_->IncomingTimestamp(frame_timestamp, last_packet_time_ms);
198    }
199  }
200  return frame;
201}
202
203void VCMReceiver::ReleaseFrame(VCMEncodedFrame* frame) {
204  jitter_buffer_.ReleaseFrame(frame);
205}
206
207void VCMReceiver::ReceiveStatistics(uint32_t* bitrate,
208                                    uint32_t* framerate) {
209  assert(bitrate);
210  assert(framerate);
211  jitter_buffer_.IncomingRateStatistics(framerate, bitrate);
212}
213
214void VCMReceiver::ReceivedFrameCount(VCMFrameCount* frame_count) const {
215  assert(frame_count);
216  std::map<FrameType, uint32_t> counts(jitter_buffer_.FrameStatistics());
217  frame_count->numDeltaFrames = counts[kVideoFrameDelta];
218  frame_count->numKeyFrames = counts[kVideoFrameKey];
219}
220
221uint32_t VCMReceiver::DiscardedPackets() const {
222  return jitter_buffer_.num_discarded_packets();
223}
224
225void VCMReceiver::SetNackMode(VCMNackMode nackMode,
226                              int low_rtt_nack_threshold_ms,
227                              int high_rtt_nack_threshold_ms) {
228  CriticalSectionScoped cs(crit_sect_);
229  // Default to always having NACK enabled in hybrid mode.
230  jitter_buffer_.SetNackMode(nackMode, low_rtt_nack_threshold_ms,
231                             high_rtt_nack_threshold_ms);
232  if (!master_) {
233    state_ = kPassive;  // The dual decoder defaults to passive.
234  }
235}
236
237void VCMReceiver::SetNackSettings(size_t max_nack_list_size,
238                                  int max_packet_age_to_nack,
239                                  int max_incomplete_time_ms) {
240  jitter_buffer_.SetNackSettings(max_nack_list_size,
241                                 max_packet_age_to_nack,
242                                 max_incomplete_time_ms);
243}
244
245VCMNackMode VCMReceiver::NackMode() const {
246  CriticalSectionScoped cs(crit_sect_);
247  return jitter_buffer_.nack_mode();
248}
249
250VCMNackStatus VCMReceiver::NackList(uint16_t* nack_list,
251                                    uint16_t size,
252                                    uint16_t* nack_list_length) {
253  bool request_key_frame = false;
254  uint16_t* internal_nack_list = jitter_buffer_.GetNackList(
255      nack_list_length, &request_key_frame);
256  assert(*nack_list_length <= size);
257  if (internal_nack_list != NULL && *nack_list_length > 0) {
258    memcpy(nack_list, internal_nack_list, *nack_list_length * sizeof(uint16_t));
259  }
260  if (request_key_frame) {
261    return kNackKeyFrameRequest;
262  }
263  return kNackOk;
264}
265
266// Decide whether we should change decoder state. This should be done if the
267// dual decoder has caught up with the decoder decoding with packet losses.
268bool VCMReceiver::DualDecoderCaughtUp(VCMEncodedFrame* dual_frame,
269                                      VCMReceiver& dual_receiver) const {
270  if (dual_frame == NULL) {
271    return false;
272  }
273  if (jitter_buffer_.LastDecodedTimestamp() == dual_frame->TimeStamp()) {
274    dual_receiver.UpdateState(kWaitForPrimaryDecode);
275    return true;
276  }
277  return false;
278}
279
280void VCMReceiver::CopyJitterBufferStateFromReceiver(
281    const VCMReceiver& receiver) {
282  jitter_buffer_.CopyFrom(receiver.jitter_buffer_);
283}
284
285VCMReceiverState VCMReceiver::State() const {
286  CriticalSectionScoped cs(crit_sect_);
287  return state_;
288}
289
290void VCMReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
291  jitter_buffer_.SetDecodeErrorMode(decode_error_mode);
292}
293
294VCMDecodeErrorMode VCMReceiver::DecodeErrorMode() const {
295  return jitter_buffer_.decode_error_mode();
296}
297
298int VCMReceiver::SetMinReceiverDelay(int desired_delay_ms) {
299  CriticalSectionScoped cs(crit_sect_);
300  if (desired_delay_ms < 0 || desired_delay_ms > kMaxReceiverDelayMs) {
301    return -1;
302  }
303  max_video_delay_ms_ = desired_delay_ms + kMaxVideoDelayMs;
304  // Initializing timing to the desired delay.
305  timing_->set_min_playout_delay(desired_delay_ms);
306  return 0;
307}
308
309int VCMReceiver::RenderBufferSizeMs() {
310  uint32_t timestamp_start = 0u;
311  uint32_t timestamp_end = 0u;
312  // Render timestamps are computed just prior to decoding. Therefore this is
313  // only an estimate based on frames' timestamps and current timing state.
314  jitter_buffer_.RenderBufferSize(&timestamp_start, &timestamp_end);
315  if (timestamp_start == timestamp_end) {
316    return 0;
317  }
318  // Update timing.
319  const int64_t now_ms = clock_->TimeInMilliseconds();
320  timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs());
321  // Get render timestamps.
322  uint32_t render_start = timing_->RenderTimeMs(timestamp_start, now_ms);
323  uint32_t render_end = timing_->RenderTimeMs(timestamp_end, now_ms);
324  return render_end - render_start;
325}
326
327void VCMReceiver::UpdateState(VCMReceiverState new_state) {
328  CriticalSectionScoped cs(crit_sect_);
329  assert(!(state_ == kPassive && new_state == kWaitForPrimaryDecode));
330  state_ = new_state;
331}
332
333void VCMReceiver::UpdateState(const VCMEncodedFrame& frame) {
334  if (jitter_buffer_.nack_mode() == kNoNack) {
335    // Dual decoder mode has not been enabled.
336    return;
337  }
338  // Update the dual receiver state.
339  if (frame.Complete() && frame.FrameType() == kVideoFrameKey) {
340    UpdateState(kPassive);
341  }
342  if (State() == kWaitForPrimaryDecode &&
343      frame.Complete() && !frame.MissingFrame()) {
344    UpdateState(kPassive);
345  }
346  if (frame.MissingFrame() || !frame.Complete()) {
347    // State was corrupted, enable dual receiver.
348    UpdateState(kReceiving);
349  }
350}
351}  // namespace webrtc
352