1/* 2 * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#include "webrtc/modules/video_coding/utility/frame_dropper.h" 12 13#include "webrtc/system_wrappers/include/trace.h" 14 15namespace webrtc { 16 17const float kDefaultKeyFrameSizeAvgKBits = 0.9f; 18const float kDefaultKeyFrameRatio = 0.99f; 19const float kDefaultDropRatioAlpha = 0.9f; 20const float kDefaultDropRatioMax = 0.96f; 21const float kDefaultMaxTimeToDropFrames = 4.0f; // In seconds. 22 23FrameDropper::FrameDropper() 24 : _keyFrameSizeAvgKbits(kDefaultKeyFrameSizeAvgKBits), 25 _keyFrameRatio(kDefaultKeyFrameRatio), 26 _dropRatio(kDefaultDropRatioAlpha, kDefaultDropRatioMax), 27 _enabled(true), 28 _max_time_drops(kDefaultMaxTimeToDropFrames) { 29 Reset(); 30} 31 32FrameDropper::FrameDropper(float max_time_drops) 33 : _keyFrameSizeAvgKbits(kDefaultKeyFrameSizeAvgKBits), 34 _keyFrameRatio(kDefaultKeyFrameRatio), 35 _dropRatio(kDefaultDropRatioAlpha, kDefaultDropRatioMax), 36 _enabled(true), 37 _max_time_drops(max_time_drops) { 38 Reset(); 39} 40 41void FrameDropper::Reset() { 42 _keyFrameRatio.Reset(0.99f); 43 _keyFrameRatio.Apply( 44 1.0f, 1.0f / 300.0f); // 1 key frame every 10th second in 30 fps 45 _keyFrameSizeAvgKbits.Reset(0.9f); 46 _keyFrameCount = 0; 47 _accumulator = 0.0f; 48 _accumulatorMax = 150.0f; // assume 300 kb/s and 0.5 s window 49 _targetBitRate = 300.0f; 50 _incoming_frame_rate = 30; 51 _keyFrameSpreadFrames = 0.5f * _incoming_frame_rate; 52 _dropNext = false; 53 _dropRatio.Reset(0.9f); 54 _dropRatio.Apply(0.0f, 0.0f); // Initialize to 0 55 _dropCount = 0; 56 _windowSize = 0.5f; 57 _wasBelowMax = true; 58 _fastMode = false; // start with normal (non-aggressive) mode 59 // Cap for the encoder buffer level/accumulator, in secs. 60 _cap_buffer_size = 3.0f; 61 // Cap on maximum amount of dropped frames between kept frames, in secs. 62 _max_time_drops = 4.0f; 63} 64 65void FrameDropper::Enable(bool enable) { 66 _enabled = enable; 67} 68 69void FrameDropper::Fill(size_t frameSizeBytes, bool deltaFrame) { 70 if (!_enabled) { 71 return; 72 } 73 float frameSizeKbits = 8.0f * static_cast<float>(frameSizeBytes) / 1000.0f; 74 if (!deltaFrame && 75 !_fastMode) { // fast mode does not treat key-frames any different 76 _keyFrameSizeAvgKbits.Apply(1, frameSizeKbits); 77 _keyFrameRatio.Apply(1.0, 1.0); 78 if (frameSizeKbits > _keyFrameSizeAvgKbits.filtered()) { 79 // Remove the average key frame size since we 80 // compensate for key frames when adding delta 81 // frames. 82 frameSizeKbits -= _keyFrameSizeAvgKbits.filtered(); 83 } else { 84 // Shouldn't be negative, so zero is the lower bound. 85 frameSizeKbits = 0; 86 } 87 if (_keyFrameRatio.filtered() > 1e-5 && 88 1 / _keyFrameRatio.filtered() < _keyFrameSpreadFrames) { 89 // We are sending key frames more often than our upper bound for 90 // how much we allow the key frame compensation to be spread 91 // out in time. Therefor we must use the key frame ratio rather 92 // than keyFrameSpreadFrames. 93 _keyFrameCount = 94 static_cast<int32_t>(1 / _keyFrameRatio.filtered() + 0.5); 95 } else { 96 // Compensate for the key frame the following frames 97 _keyFrameCount = static_cast<int32_t>(_keyFrameSpreadFrames + 0.5); 98 } 99 } else { 100 // Decrease the keyFrameRatio 101 _keyFrameRatio.Apply(1.0, 0.0); 102 } 103 // Change the level of the accumulator (bucket) 104 _accumulator += frameSizeKbits; 105 CapAccumulator(); 106} 107 108void FrameDropper::Leak(uint32_t inputFrameRate) { 109 if (!_enabled) { 110 return; 111 } 112 if (inputFrameRate < 1) { 113 return; 114 } 115 if (_targetBitRate < 0.0f) { 116 return; 117 } 118 _keyFrameSpreadFrames = 0.5f * inputFrameRate; 119 // T is the expected bits per frame (target). If all frames were the same 120 // size, 121 // we would get T bits per frame. Notice that T is also weighted to be able to 122 // force a lower frame rate if wanted. 123 float T = _targetBitRate / inputFrameRate; 124 if (_keyFrameCount > 0) { 125 // Perform the key frame compensation 126 if (_keyFrameRatio.filtered() > 0 && 127 1 / _keyFrameRatio.filtered() < _keyFrameSpreadFrames) { 128 T -= _keyFrameSizeAvgKbits.filtered() * _keyFrameRatio.filtered(); 129 } else { 130 T -= _keyFrameSizeAvgKbits.filtered() / _keyFrameSpreadFrames; 131 } 132 _keyFrameCount--; 133 } 134 _accumulator -= T; 135 if (_accumulator < 0.0f) { 136 _accumulator = 0.0f; 137 } 138 UpdateRatio(); 139} 140 141void FrameDropper::UpdateNack(uint32_t nackBytes) { 142 if (!_enabled) { 143 return; 144 } 145 _accumulator += static_cast<float>(nackBytes) * 8.0f / 1000.0f; 146} 147 148void FrameDropper::FillBucket(float inKbits, float outKbits) { 149 _accumulator += (inKbits - outKbits); 150} 151 152void FrameDropper::UpdateRatio() { 153 if (_accumulator > 1.3f * _accumulatorMax) { 154 // Too far above accumulator max, react faster 155 _dropRatio.UpdateBase(0.8f); 156 } else { 157 // Go back to normal reaction 158 _dropRatio.UpdateBase(0.9f); 159 } 160 if (_accumulator > _accumulatorMax) { 161 // We are above accumulator max, and should ideally 162 // drop a frame. Increase the dropRatio and drop 163 // the frame later. 164 if (_wasBelowMax) { 165 _dropNext = true; 166 } 167 if (_fastMode) { 168 // always drop in aggressive mode 169 _dropNext = true; 170 } 171 172 _dropRatio.Apply(1.0f, 1.0f); 173 _dropRatio.UpdateBase(0.9f); 174 } else { 175 _dropRatio.Apply(1.0f, 0.0f); 176 } 177 _wasBelowMax = _accumulator < _accumulatorMax; 178} 179 180// This function signals when to drop frames to the caller. It makes use of the 181// dropRatio 182// to smooth out the drops over time. 183bool FrameDropper::DropFrame() { 184 if (!_enabled) { 185 return false; 186 } 187 if (_dropNext) { 188 _dropNext = false; 189 _dropCount = 0; 190 } 191 192 if (_dropRatio.filtered() >= 0.5f) { // Drops per keep 193 // limit is the number of frames we should drop between each kept frame 194 // to keep our drop ratio. limit is positive in this case. 195 float denom = 1.0f - _dropRatio.filtered(); 196 if (denom < 1e-5) { 197 denom = 1e-5f; 198 } 199 int32_t limit = static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f); 200 // Put a bound on the max amount of dropped frames between each kept 201 // frame, in terms of frame rate and window size (secs). 202 int max_limit = static_cast<int>(_incoming_frame_rate * _max_time_drops); 203 if (limit > max_limit) { 204 limit = max_limit; 205 } 206 if (_dropCount < 0) { 207 // Reset the _dropCount since it was negative and should be positive. 208 if (_dropRatio.filtered() > 0.4f) { 209 _dropCount = -_dropCount; 210 } else { 211 _dropCount = 0; 212 } 213 } 214 if (_dropCount < limit) { 215 // As long we are below the limit we should drop frames. 216 _dropCount++; 217 return true; 218 } else { 219 // Only when we reset _dropCount a frame should be kept. 220 _dropCount = 0; 221 return false; 222 } 223 } else if (_dropRatio.filtered() > 0.0f && 224 _dropRatio.filtered() < 0.5f) { // Keeps per drop 225 // limit is the number of frames we should keep between each drop 226 // in order to keep the drop ratio. limit is negative in this case, 227 // and the _dropCount is also negative. 228 float denom = _dropRatio.filtered(); 229 if (denom < 1e-5) { 230 denom = 1e-5f; 231 } 232 int32_t limit = -static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f); 233 if (_dropCount > 0) { 234 // Reset the _dropCount since we have a positive 235 // _dropCount, and it should be negative. 236 if (_dropRatio.filtered() < 0.6f) { 237 _dropCount = -_dropCount; 238 } else { 239 _dropCount = 0; 240 } 241 } 242 if (_dropCount > limit) { 243 if (_dropCount == 0) { 244 // Drop frames when we reset _dropCount. 245 _dropCount--; 246 return true; 247 } else { 248 // Keep frames as long as we haven't reached limit. 249 _dropCount--; 250 return false; 251 } 252 } else { 253 _dropCount = 0; 254 return false; 255 } 256 } 257 _dropCount = 0; 258 return false; 259 260 // A simpler version, unfiltered and quicker 261 // bool dropNext = _dropNext; 262 // _dropNext = false; 263 // return dropNext; 264} 265 266void FrameDropper::SetRates(float bitRate, float incoming_frame_rate) { 267 // Bit rate of -1 means infinite bandwidth. 268 _accumulatorMax = bitRate * _windowSize; // bitRate * windowSize (in seconds) 269 if (_targetBitRate > 0.0f && bitRate < _targetBitRate && 270 _accumulator > _accumulatorMax) { 271 // Rescale the accumulator level if the accumulator max decreases 272 _accumulator = bitRate / _targetBitRate * _accumulator; 273 } 274 _targetBitRate = bitRate; 275 CapAccumulator(); 276 _incoming_frame_rate = incoming_frame_rate; 277} 278 279float FrameDropper::ActualFrameRate(uint32_t inputFrameRate) const { 280 if (!_enabled) { 281 return static_cast<float>(inputFrameRate); 282 } 283 return inputFrameRate * (1.0f - _dropRatio.filtered()); 284} 285 286// Put a cap on the accumulator, i.e., don't let it grow beyond some level. 287// This is a temporary fix for screencasting where very large frames from 288// encoder will cause very slow response (too many frame drops). 289void FrameDropper::CapAccumulator() { 290 float max_accumulator = _targetBitRate * _cap_buffer_size; 291 if (_accumulator > max_accumulator) { 292 _accumulator = max_accumulator; 293 } 294} 295} // namespace webrtc 296