1/*
2 *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/modules/video_coding/utility/include/frame_dropper.h"
12
13#include "webrtc/system_wrappers/interface/trace.h"
14
15namespace webrtc
16{
17
18const float kDefaultKeyFrameSizeAvgKBits = 0.9f;
19const float kDefaultKeyFrameRatio = 0.99f;
20const float kDefaultDropRatioAlpha = 0.9f;
21const float kDefaultDropRatioMax = 0.96f;
22const float kDefaultMaxTimeToDropFrames = 4.0f;  // In seconds.
23
24FrameDropper::FrameDropper()
25:
26_keyFrameSizeAvgKbits(kDefaultKeyFrameSizeAvgKBits),
27_keyFrameRatio(kDefaultKeyFrameRatio),
28_dropRatio(kDefaultDropRatioAlpha, kDefaultDropRatioMax),
29_enabled(true),
30_max_time_drops(kDefaultMaxTimeToDropFrames)
31{
32    Reset();
33}
34
35FrameDropper::FrameDropper(float max_time_drops)
36:
37_keyFrameSizeAvgKbits(kDefaultKeyFrameSizeAvgKBits),
38_keyFrameRatio(kDefaultKeyFrameRatio),
39_dropRatio(kDefaultDropRatioAlpha, kDefaultDropRatioMax),
40_enabled(true),
41_max_time_drops(max_time_drops)
42{
43    Reset();
44}
45
46void
47FrameDropper::Reset()
48{
49    _keyFrameRatio.Reset(0.99f);
50    _keyFrameRatio.Apply(1.0f, 1.0f/300.0f); // 1 key frame every 10th second in 30 fps
51    _keyFrameSizeAvgKbits.Reset(0.9f);
52    _keyFrameCount = 0;
53    _accumulator = 0.0f;
54    _accumulatorMax = 150.0f; // assume 300 kb/s and 0.5 s window
55    _targetBitRate = 300.0f;
56    _incoming_frame_rate = 30;
57    _keyFrameSpreadFrames = 0.5f * _incoming_frame_rate;
58    _dropNext = false;
59    _dropRatio.Reset(0.9f);
60    _dropRatio.Apply(0.0f, 0.0f); // Initialize to 0
61    _dropCount = 0;
62    _windowSize = 0.5f;
63    _wasBelowMax = true;
64    _fastMode = false; // start with normal (non-aggressive) mode
65    // Cap for the encoder buffer level/accumulator, in secs.
66    _cap_buffer_size = 3.0f;
67    // Cap on maximum amount of dropped frames between kept frames, in secs.
68    _max_time_drops = 4.0f;
69}
70
71void
72FrameDropper::Enable(bool enable)
73{
74    _enabled = enable;
75}
76
77void
78FrameDropper::Fill(uint32_t frameSizeBytes, bool deltaFrame)
79{
80    if (!_enabled)
81    {
82        return;
83    }
84    float frameSizeKbits = 8.0f * static_cast<float>(frameSizeBytes) / 1000.0f;
85    if (!deltaFrame && !_fastMode) // fast mode does not treat key-frames any different
86    {
87        _keyFrameSizeAvgKbits.Apply(1, frameSizeKbits);
88        _keyFrameRatio.Apply(1.0, 1.0);
89        if (frameSizeKbits > _keyFrameSizeAvgKbits.filtered())
90        {
91            // Remove the average key frame size since we
92            // compensate for key frames when adding delta
93            // frames.
94            frameSizeKbits -= _keyFrameSizeAvgKbits.filtered();
95        }
96        else
97        {
98            // Shouldn't be negative, so zero is the lower bound.
99            frameSizeKbits = 0;
100        }
101        if (_keyFrameRatio.filtered() > 1e-5 &&
102            1 / _keyFrameRatio.filtered() < _keyFrameSpreadFrames)
103        {
104            // We are sending key frames more often than our upper bound for
105            // how much we allow the key frame compensation to be spread
106            // out in time. Therefor we must use the key frame ratio rather
107            // than keyFrameSpreadFrames.
108            _keyFrameCount =
109                static_cast<int32_t>(1 / _keyFrameRatio.filtered() + 0.5);
110        }
111        else
112        {
113            // Compensate for the key frame the following frames
114            _keyFrameCount = static_cast<int32_t>(_keyFrameSpreadFrames + 0.5);
115        }
116    }
117    else
118    {
119        // Decrease the keyFrameRatio
120        _keyFrameRatio.Apply(1.0, 0.0);
121    }
122    // Change the level of the accumulator (bucket)
123    _accumulator += frameSizeKbits;
124    CapAccumulator();
125}
126
127void
128FrameDropper::Leak(uint32_t inputFrameRate)
129{
130    if (!_enabled)
131    {
132        return;
133    }
134    if (inputFrameRate < 1)
135    {
136        return;
137    }
138    if (_targetBitRate < 0.0f)
139    {
140        return;
141    }
142    _keyFrameSpreadFrames = 0.5f * inputFrameRate;
143    // T is the expected bits per frame (target). If all frames were the same size,
144    // we would get T bits per frame. Notice that T is also weighted to be able to
145    // force a lower frame rate if wanted.
146    float T = _targetBitRate / inputFrameRate;
147    if (_keyFrameCount > 0)
148    {
149        // Perform the key frame compensation
150        if (_keyFrameRatio.filtered() > 0 &&
151            1 / _keyFrameRatio.filtered() < _keyFrameSpreadFrames)
152        {
153            T -= _keyFrameSizeAvgKbits.filtered() * _keyFrameRatio.filtered();
154        }
155        else
156        {
157            T -= _keyFrameSizeAvgKbits.filtered() / _keyFrameSpreadFrames;
158        }
159        _keyFrameCount--;
160    }
161    _accumulator -= T;
162    if (_accumulator < 0.0f)
163    {
164        _accumulator = 0.0f;
165    }
166    UpdateRatio();
167}
168
169void
170FrameDropper::UpdateNack(uint32_t nackBytes)
171{
172    if (!_enabled)
173    {
174        return;
175    }
176    _accumulator += static_cast<float>(nackBytes) * 8.0f / 1000.0f;
177}
178
179void
180FrameDropper::FillBucket(float inKbits, float outKbits)
181{
182    _accumulator += (inKbits - outKbits);
183}
184
185void
186FrameDropper::UpdateRatio()
187{
188    if (_accumulator > 1.3f * _accumulatorMax)
189    {
190        // Too far above accumulator max, react faster
191        _dropRatio.UpdateBase(0.8f);
192    }
193    else
194    {
195        // Go back to normal reaction
196        _dropRatio.UpdateBase(0.9f);
197    }
198    if (_accumulator > _accumulatorMax)
199    {
200        // We are above accumulator max, and should ideally
201        // drop a frame. Increase the dropRatio and drop
202        // the frame later.
203        if (_wasBelowMax)
204        {
205            _dropNext = true;
206        }
207        if (_fastMode)
208        {
209            // always drop in aggressive mode
210            _dropNext = true;
211        }
212
213        _dropRatio.Apply(1.0f, 1.0f);
214        _dropRatio.UpdateBase(0.9f);
215    }
216    else
217    {
218        _dropRatio.Apply(1.0f, 0.0f);
219    }
220    _wasBelowMax = _accumulator < _accumulatorMax;
221}
222
223// This function signals when to drop frames to the caller. It makes use of the dropRatio
224// to smooth out the drops over time.
225bool
226FrameDropper::DropFrame()
227{
228    if (!_enabled)
229    {
230        return false;
231    }
232    if (_dropNext)
233    {
234        _dropNext = false;
235        _dropCount = 0;
236    }
237
238    if (_dropRatio.filtered() >= 0.5f) // Drops per keep
239    {
240        // limit is the number of frames we should drop between each kept frame
241        // to keep our drop ratio. limit is positive in this case.
242        float denom = 1.0f - _dropRatio.filtered();
243        if (denom < 1e-5)
244        {
245            denom = (float)1e-5;
246        }
247        int32_t limit = static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f);
248        // Put a bound on the max amount of dropped frames between each kept
249        // frame, in terms of frame rate and window size (secs).
250        int max_limit = static_cast<int>(_incoming_frame_rate *
251                                         _max_time_drops);
252        if (limit > max_limit) {
253          limit = max_limit;
254        }
255        if (_dropCount < 0)
256        {
257            // Reset the _dropCount since it was negative and should be positive.
258            if (_dropRatio.filtered() > 0.4f)
259            {
260                _dropCount = -_dropCount;
261            }
262            else
263            {
264                _dropCount = 0;
265            }
266        }
267        if (_dropCount < limit)
268        {
269            // As long we are below the limit we should drop frames.
270            _dropCount++;
271            return true;
272        }
273        else
274        {
275            // Only when we reset _dropCount a frame should be kept.
276            _dropCount = 0;
277            return false;
278        }
279    }
280    else if (_dropRatio.filtered() > 0.0f &&
281        _dropRatio.filtered() < 0.5f) // Keeps per drop
282    {
283        // limit is the number of frames we should keep between each drop
284        // in order to keep the drop ratio. limit is negative in this case,
285        // and the _dropCount is also negative.
286        float denom = _dropRatio.filtered();
287        if (denom < 1e-5)
288        {
289            denom = (float)1e-5;
290        }
291        int32_t limit = -static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f);
292        if (_dropCount > 0)
293        {
294            // Reset the _dropCount since we have a positive
295            // _dropCount, and it should be negative.
296            if (_dropRatio.filtered() < 0.6f)
297            {
298                _dropCount = -_dropCount;
299            }
300            else
301            {
302                _dropCount = 0;
303            }
304        }
305        if (_dropCount > limit)
306        {
307            if (_dropCount == 0)
308            {
309                // Drop frames when we reset _dropCount.
310                _dropCount--;
311                return true;
312            }
313            else
314            {
315                // Keep frames as long as we haven't reached limit.
316                _dropCount--;
317                return false;
318            }
319        }
320        else
321        {
322            _dropCount = 0;
323            return false;
324        }
325    }
326    _dropCount = 0;
327    return false;
328
329    // A simpler version, unfiltered and quicker
330    //bool dropNext = _dropNext;
331    //_dropNext = false;
332    //return dropNext;
333}
334
335void
336FrameDropper::SetRates(float bitRate, float incoming_frame_rate)
337{
338    // Bit rate of -1 means infinite bandwidth.
339    _accumulatorMax = bitRate * _windowSize; // bitRate * windowSize (in seconds)
340    if (_targetBitRate > 0.0f && bitRate < _targetBitRate && _accumulator > _accumulatorMax)
341    {
342        // Rescale the accumulator level if the accumulator max decreases
343        _accumulator = bitRate / _targetBitRate * _accumulator;
344    }
345    _targetBitRate = bitRate;
346    CapAccumulator();
347    _incoming_frame_rate = incoming_frame_rate;
348}
349
350float
351FrameDropper::ActualFrameRate(uint32_t inputFrameRate) const
352{
353    if (!_enabled)
354    {
355        return static_cast<float>(inputFrameRate);
356    }
357    return inputFrameRate * (1.0f - _dropRatio.filtered());
358}
359
360// Put a cap on the accumulator, i.e., don't let it grow beyond some level.
361// This is a temporary fix for screencasting where very large frames from
362// encoder will cause very slow response (too many frame drops).
363void FrameDropper::CapAccumulator() {
364  float max_accumulator = _targetBitRate * _cap_buffer_size;
365  if (_accumulator > max_accumulator) {
366    _accumulator = max_accumulator;
367  }
368}
369
370}
371