1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/video/stream_synchronization.h"
12
13#include <assert.h>
14#include <math.h>
15#include <stdlib.h>
16
17#include <algorithm>
18
19#include "webrtc/base/logging.h"
20
21namespace webrtc {
22
23static const int kMaxChangeMs = 80;
24static const int kMaxDeltaDelayMs = 10000;
25static const int kFilterLength = 4;
26// Minimum difference between audio and video to warrant a change.
27static const int kMinDeltaMs = 30;
28
29struct ViESyncDelay {
30  ViESyncDelay() {
31    extra_video_delay_ms = 0;
32    last_video_delay_ms = 0;
33    extra_audio_delay_ms = 0;
34    last_audio_delay_ms = 0;
35    network_delay = 120;
36  }
37
38  int extra_video_delay_ms;
39  int last_video_delay_ms;
40  int extra_audio_delay_ms;
41  int last_audio_delay_ms;
42  int network_delay;
43};
44
45StreamSynchronization::StreamSynchronization(uint32_t video_primary_ssrc,
46                                             int audio_channel_id)
47    : channel_delay_(new ViESyncDelay),
48      video_primary_ssrc_(video_primary_ssrc),
49      audio_channel_id_(audio_channel_id),
50      base_target_delay_ms_(0),
51      avg_diff_ms_(0) {
52}
53
54StreamSynchronization::~StreamSynchronization() {
55  delete channel_delay_;
56}
57
58bool StreamSynchronization::ComputeRelativeDelay(
59    const Measurements& audio_measurement,
60    const Measurements& video_measurement,
61    int* relative_delay_ms) {
62  assert(relative_delay_ms);
63  if (audio_measurement.rtcp.size() < 2 || video_measurement.rtcp.size() < 2) {
64    // We need two RTCP SR reports per stream to do synchronization.
65    return false;
66  }
67  int64_t audio_last_capture_time_ms;
68  if (!RtpToNtpMs(audio_measurement.latest_timestamp,
69                  audio_measurement.rtcp,
70                  &audio_last_capture_time_ms)) {
71    return false;
72  }
73  int64_t video_last_capture_time_ms;
74  if (!RtpToNtpMs(video_measurement.latest_timestamp,
75                  video_measurement.rtcp,
76                  &video_last_capture_time_ms)) {
77    return false;
78  }
79  if (video_last_capture_time_ms < 0) {
80    return false;
81  }
82  // Positive diff means that video_measurement is behind audio_measurement.
83  *relative_delay_ms = video_measurement.latest_receive_time_ms -
84      audio_measurement.latest_receive_time_ms -
85      (video_last_capture_time_ms - audio_last_capture_time_ms);
86  if (*relative_delay_ms > kMaxDeltaDelayMs ||
87      *relative_delay_ms < -kMaxDeltaDelayMs) {
88    return false;
89  }
90  return true;
91}
92
93bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
94                                          int current_audio_delay_ms,
95                                          int* total_audio_delay_target_ms,
96                                          int* total_video_delay_target_ms) {
97  assert(total_audio_delay_target_ms && total_video_delay_target_ms);
98
99  int current_video_delay_ms = *total_video_delay_target_ms;
100  LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms
101                  << ", network delay diff: " << channel_delay_->network_delay
102                  << " current diff: " << relative_delay_ms
103                  << " for channel " << audio_channel_id_;
104  // Calculate the difference between the lowest possible video delay and
105  // the current audio delay.
106  int current_diff_ms = current_video_delay_ms - current_audio_delay_ms +
107      relative_delay_ms;
108
109  avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ +
110      current_diff_ms) / kFilterLength;
111  if (abs(avg_diff_ms_) < kMinDeltaMs) {
112    // Don't adjust if the diff is within our margin.
113    return false;
114  }
115
116  // Make sure we don't move too fast.
117  int diff_ms = avg_diff_ms_ / 2;
118  diff_ms = std::min(diff_ms, kMaxChangeMs);
119  diff_ms = std::max(diff_ms, -kMaxChangeMs);
120
121  // Reset the average after a move to prevent overshooting reaction.
122  avg_diff_ms_ = 0;
123
124  if (diff_ms > 0) {
125    // The minimum video delay is longer than the current audio delay.
126    // We need to decrease extra video delay, or add extra audio delay.
127    if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) {
128      // We have extra delay added to ViE. Reduce this delay before adding
129      // extra delay to VoE.
130      channel_delay_->extra_video_delay_ms -= diff_ms;
131      channel_delay_->extra_audio_delay_ms = base_target_delay_ms_;
132    } else {  // channel_delay_->extra_video_delay_ms > 0
133      // We have no extra video delay to remove, increase the audio delay.
134      channel_delay_->extra_audio_delay_ms += diff_ms;
135      channel_delay_->extra_video_delay_ms = base_target_delay_ms_;
136    }
137  } else {  // if (diff_ms > 0)
138    // The video delay is lower than the current audio delay.
139    // We need to decrease extra audio delay, or add extra video delay.
140    if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) {
141      // We have extra delay in VoiceEngine.
142      // Start with decreasing the voice delay.
143      // Note: diff_ms is negative; add the negative difference.
144      channel_delay_->extra_audio_delay_ms += diff_ms;
145      channel_delay_->extra_video_delay_ms = base_target_delay_ms_;
146    } else {  // channel_delay_->extra_audio_delay_ms > base_target_delay_ms_
147      // We have no extra delay in VoiceEngine, increase the video delay.
148      // Note: diff_ms is negative; subtract the negative difference.
149      channel_delay_->extra_video_delay_ms -= diff_ms;  // X - (-Y) = X + Y.
150      channel_delay_->extra_audio_delay_ms = base_target_delay_ms_;
151    }
152  }
153
154  // Make sure that video is never below our target.
155  channel_delay_->extra_video_delay_ms = std::max(
156      channel_delay_->extra_video_delay_ms, base_target_delay_ms_);
157
158  int new_video_delay_ms;
159  if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) {
160    new_video_delay_ms = channel_delay_->extra_video_delay_ms;
161  } else {
162    // No change to the extra video delay. We are changing audio and we only
163    // allow to change one at the time.
164    new_video_delay_ms = channel_delay_->last_video_delay_ms;
165  }
166
167  // Make sure that we don't go below the extra video delay.
168  new_video_delay_ms = std::max(
169      new_video_delay_ms, channel_delay_->extra_video_delay_ms);
170
171  // Verify we don't go above the maximum allowed video delay.
172  new_video_delay_ms =
173      std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
174
175  int new_audio_delay_ms;
176  if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) {
177    new_audio_delay_ms = channel_delay_->extra_audio_delay_ms;
178  } else {
179    // No change to the audio delay. We are changing video and we only
180    // allow to change one at the time.
181    new_audio_delay_ms = channel_delay_->last_audio_delay_ms;
182  }
183
184  // Make sure that we don't go below the extra audio delay.
185  new_audio_delay_ms = std::max(
186      new_audio_delay_ms, channel_delay_->extra_audio_delay_ms);
187
188  // Verify we don't go above the maximum allowed audio delay.
189  new_audio_delay_ms =
190      std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
191
192  // Remember our last audio and video delays.
193  channel_delay_->last_video_delay_ms = new_video_delay_ms;
194  channel_delay_->last_audio_delay_ms = new_audio_delay_ms;
195
196  LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms
197                  << " for video primary SSRC " << video_primary_ssrc_
198                  << " and audio delay " << channel_delay_->extra_audio_delay_ms
199                  << " for audio channel " << audio_channel_id_;
200
201  // Return values.
202  *total_video_delay_target_ms = new_video_delay_ms;
203  *total_audio_delay_target_ms = new_audio_delay_ms;
204  return true;
205}
206
207void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) {
208  // Initial extra delay for audio (accounting for existing extra delay).
209  channel_delay_->extra_audio_delay_ms +=
210      target_delay_ms - base_target_delay_ms_;
211  channel_delay_->last_audio_delay_ms +=
212      target_delay_ms - base_target_delay_ms_;
213
214  // The video delay is compared to the last value (and how much we can update
215  // is limited by that as well).
216  channel_delay_->last_video_delay_ms +=
217      target_delay_ms - base_target_delay_ms_;
218
219  channel_delay_->extra_video_delay_ms +=
220      target_delay_ms - base_target_delay_ms_;
221
222  // Video is already delayed by the desired amount.
223  base_target_delay_ms_ = target_delay_ms;
224}
225
226}  // namespace webrtc
227