1/*
2 *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/modules/video_coding/utility/frame_dropper.h"
12
13#include "webrtc/system_wrappers/include/trace.h"
14
15namespace webrtc {
16
17const float kDefaultKeyFrameSizeAvgKBits = 0.9f;
18const float kDefaultKeyFrameRatio = 0.99f;
19const float kDefaultDropRatioAlpha = 0.9f;
20const float kDefaultDropRatioMax = 0.96f;
21const float kDefaultMaxTimeToDropFrames = 4.0f;  // In seconds.
22
23FrameDropper::FrameDropper()
24    : _keyFrameSizeAvgKbits(kDefaultKeyFrameSizeAvgKBits),
25      _keyFrameRatio(kDefaultKeyFrameRatio),
26      _dropRatio(kDefaultDropRatioAlpha, kDefaultDropRatioMax),
27      _enabled(true),
28      _max_time_drops(kDefaultMaxTimeToDropFrames) {
29  Reset();
30}
31
32FrameDropper::FrameDropper(float max_time_drops)
33    : _keyFrameSizeAvgKbits(kDefaultKeyFrameSizeAvgKBits),
34      _keyFrameRatio(kDefaultKeyFrameRatio),
35      _dropRatio(kDefaultDropRatioAlpha, kDefaultDropRatioMax),
36      _enabled(true),
37      _max_time_drops(max_time_drops) {
38  Reset();
39}
40
41void FrameDropper::Reset() {
42  _keyFrameRatio.Reset(0.99f);
43  _keyFrameRatio.Apply(
44      1.0f, 1.0f / 300.0f);  // 1 key frame every 10th second in 30 fps
45  _keyFrameSizeAvgKbits.Reset(0.9f);
46  _keyFrameCount = 0;
47  _accumulator = 0.0f;
48  _accumulatorMax = 150.0f;  // assume 300 kb/s and 0.5 s window
49  _targetBitRate = 300.0f;
50  _incoming_frame_rate = 30;
51  _keyFrameSpreadFrames = 0.5f * _incoming_frame_rate;
52  _dropNext = false;
53  _dropRatio.Reset(0.9f);
54  _dropRatio.Apply(0.0f, 0.0f);  // Initialize to 0
55  _dropCount = 0;
56  _windowSize = 0.5f;
57  _wasBelowMax = true;
58  _fastMode = false;  // start with normal (non-aggressive) mode
59  // Cap for the encoder buffer level/accumulator, in secs.
60  _cap_buffer_size = 3.0f;
61  // Cap on maximum amount of dropped frames between kept frames, in secs.
62  _max_time_drops = 4.0f;
63}
64
65void FrameDropper::Enable(bool enable) {
66  _enabled = enable;
67}
68
69void FrameDropper::Fill(size_t frameSizeBytes, bool deltaFrame) {
70  if (!_enabled) {
71    return;
72  }
73  float frameSizeKbits = 8.0f * static_cast<float>(frameSizeBytes) / 1000.0f;
74  if (!deltaFrame &&
75      !_fastMode) {  // fast mode does not treat key-frames any different
76    _keyFrameSizeAvgKbits.Apply(1, frameSizeKbits);
77    _keyFrameRatio.Apply(1.0, 1.0);
78    if (frameSizeKbits > _keyFrameSizeAvgKbits.filtered()) {
79      // Remove the average key frame size since we
80      // compensate for key frames when adding delta
81      // frames.
82      frameSizeKbits -= _keyFrameSizeAvgKbits.filtered();
83    } else {
84      // Shouldn't be negative, so zero is the lower bound.
85      frameSizeKbits = 0;
86    }
87    if (_keyFrameRatio.filtered() > 1e-5 &&
88        1 / _keyFrameRatio.filtered() < _keyFrameSpreadFrames) {
89      // We are sending key frames more often than our upper bound for
90      // how much we allow the key frame compensation to be spread
91      // out in time. Therefor we must use the key frame ratio rather
92      // than keyFrameSpreadFrames.
93      _keyFrameCount =
94          static_cast<int32_t>(1 / _keyFrameRatio.filtered() + 0.5);
95    } else {
96      // Compensate for the key frame the following frames
97      _keyFrameCount = static_cast<int32_t>(_keyFrameSpreadFrames + 0.5);
98    }
99  } else {
100    // Decrease the keyFrameRatio
101    _keyFrameRatio.Apply(1.0, 0.0);
102  }
103  // Change the level of the accumulator (bucket)
104  _accumulator += frameSizeKbits;
105  CapAccumulator();
106}
107
108void FrameDropper::Leak(uint32_t inputFrameRate) {
109  if (!_enabled) {
110    return;
111  }
112  if (inputFrameRate < 1) {
113    return;
114  }
115  if (_targetBitRate < 0.0f) {
116    return;
117  }
118  _keyFrameSpreadFrames = 0.5f * inputFrameRate;
119  // T is the expected bits per frame (target). If all frames were the same
120  // size,
121  // we would get T bits per frame. Notice that T is also weighted to be able to
122  // force a lower frame rate if wanted.
123  float T = _targetBitRate / inputFrameRate;
124  if (_keyFrameCount > 0) {
125    // Perform the key frame compensation
126    if (_keyFrameRatio.filtered() > 0 &&
127        1 / _keyFrameRatio.filtered() < _keyFrameSpreadFrames) {
128      T -= _keyFrameSizeAvgKbits.filtered() * _keyFrameRatio.filtered();
129    } else {
130      T -= _keyFrameSizeAvgKbits.filtered() / _keyFrameSpreadFrames;
131    }
132    _keyFrameCount--;
133  }
134  _accumulator -= T;
135  if (_accumulator < 0.0f) {
136    _accumulator = 0.0f;
137  }
138  UpdateRatio();
139}
140
141void FrameDropper::UpdateNack(uint32_t nackBytes) {
142  if (!_enabled) {
143    return;
144  }
145  _accumulator += static_cast<float>(nackBytes) * 8.0f / 1000.0f;
146}
147
148void FrameDropper::FillBucket(float inKbits, float outKbits) {
149  _accumulator += (inKbits - outKbits);
150}
151
152void FrameDropper::UpdateRatio() {
153  if (_accumulator > 1.3f * _accumulatorMax) {
154    // Too far above accumulator max, react faster
155    _dropRatio.UpdateBase(0.8f);
156  } else {
157    // Go back to normal reaction
158    _dropRatio.UpdateBase(0.9f);
159  }
160  if (_accumulator > _accumulatorMax) {
161    // We are above accumulator max, and should ideally
162    // drop a frame. Increase the dropRatio and drop
163    // the frame later.
164    if (_wasBelowMax) {
165      _dropNext = true;
166    }
167    if (_fastMode) {
168      // always drop in aggressive mode
169      _dropNext = true;
170    }
171
172    _dropRatio.Apply(1.0f, 1.0f);
173    _dropRatio.UpdateBase(0.9f);
174  } else {
175    _dropRatio.Apply(1.0f, 0.0f);
176  }
177  _wasBelowMax = _accumulator < _accumulatorMax;
178}
179
180// This function signals when to drop frames to the caller. It makes use of the
181// dropRatio
182// to smooth out the drops over time.
183bool FrameDropper::DropFrame() {
184  if (!_enabled) {
185    return false;
186  }
187  if (_dropNext) {
188    _dropNext = false;
189    _dropCount = 0;
190  }
191
192  if (_dropRatio.filtered() >= 0.5f) {  // Drops per keep
193    // limit is the number of frames we should drop between each kept frame
194    // to keep our drop ratio. limit is positive in this case.
195    float denom = 1.0f - _dropRatio.filtered();
196    if (denom < 1e-5) {
197      denom = 1e-5f;
198    }
199    int32_t limit = static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f);
200    // Put a bound on the max amount of dropped frames between each kept
201    // frame, in terms of frame rate and window size (secs).
202    int max_limit = static_cast<int>(_incoming_frame_rate * _max_time_drops);
203    if (limit > max_limit) {
204      limit = max_limit;
205    }
206    if (_dropCount < 0) {
207      // Reset the _dropCount since it was negative and should be positive.
208      if (_dropRatio.filtered() > 0.4f) {
209        _dropCount = -_dropCount;
210      } else {
211        _dropCount = 0;
212      }
213    }
214    if (_dropCount < limit) {
215      // As long we are below the limit we should drop frames.
216      _dropCount++;
217      return true;
218    } else {
219      // Only when we reset _dropCount a frame should be kept.
220      _dropCount = 0;
221      return false;
222    }
223  } else if (_dropRatio.filtered() > 0.0f &&
224             _dropRatio.filtered() < 0.5f) {  // Keeps per drop
225    // limit is the number of frames we should keep between each drop
226    // in order to keep the drop ratio. limit is negative in this case,
227    // and the _dropCount is also negative.
228    float denom = _dropRatio.filtered();
229    if (denom < 1e-5) {
230      denom = 1e-5f;
231    }
232    int32_t limit = -static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f);
233    if (_dropCount > 0) {
234      // Reset the _dropCount since we have a positive
235      // _dropCount, and it should be negative.
236      if (_dropRatio.filtered() < 0.6f) {
237        _dropCount = -_dropCount;
238      } else {
239        _dropCount = 0;
240      }
241    }
242    if (_dropCount > limit) {
243      if (_dropCount == 0) {
244        // Drop frames when we reset _dropCount.
245        _dropCount--;
246        return true;
247      } else {
248        // Keep frames as long as we haven't reached limit.
249        _dropCount--;
250        return false;
251      }
252    } else {
253      _dropCount = 0;
254      return false;
255    }
256  }
257  _dropCount = 0;
258  return false;
259
260  // A simpler version, unfiltered and quicker
261  // bool dropNext = _dropNext;
262  // _dropNext = false;
263  // return dropNext;
264}
265
266void FrameDropper::SetRates(float bitRate, float incoming_frame_rate) {
267  // Bit rate of -1 means infinite bandwidth.
268  _accumulatorMax = bitRate * _windowSize;  // bitRate * windowSize (in seconds)
269  if (_targetBitRate > 0.0f && bitRate < _targetBitRate &&
270      _accumulator > _accumulatorMax) {
271    // Rescale the accumulator level if the accumulator max decreases
272    _accumulator = bitRate / _targetBitRate * _accumulator;
273  }
274  _targetBitRate = bitRate;
275  CapAccumulator();
276  _incoming_frame_rate = incoming_frame_rate;
277}
278
279float FrameDropper::ActualFrameRate(uint32_t inputFrameRate) const {
280  if (!_enabled) {
281    return static_cast<float>(inputFrameRate);
282  }
283  return inputFrameRate * (1.0f - _dropRatio.filtered());
284}
285
286// Put a cap on the accumulator, i.e., don't let it grow beyond some level.
287// This is a temporary fix for screencasting where very large frames from
288// encoder will cause very slow response (too many frame drops).
289void FrameDropper::CapAccumulator() {
290  float max_accumulator = _targetBitRate * _cap_buffer_size;
291  if (_accumulator > max_accumulator) {
292    _accumulator = max_accumulator;
293  }
294}
295}  // namespace webrtc
296