1/* 2 * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#include "webrtc/modules/video_coding/utility/include/frame_dropper.h" 12 13#include "webrtc/system_wrappers/interface/trace.h" 14 15namespace webrtc 16{ 17 18const float kDefaultKeyFrameSizeAvgKBits = 0.9f; 19const float kDefaultKeyFrameRatio = 0.99f; 20const float kDefaultDropRatioAlpha = 0.9f; 21const float kDefaultDropRatioMax = 0.96f; 22const float kDefaultMaxTimeToDropFrames = 4.0f; // In seconds. 23 24FrameDropper::FrameDropper() 25: 26_keyFrameSizeAvgKbits(kDefaultKeyFrameSizeAvgKBits), 27_keyFrameRatio(kDefaultKeyFrameRatio), 28_dropRatio(kDefaultDropRatioAlpha, kDefaultDropRatioMax), 29_enabled(true), 30_max_time_drops(kDefaultMaxTimeToDropFrames) 31{ 32 Reset(); 33} 34 35FrameDropper::FrameDropper(float max_time_drops) 36: 37_keyFrameSizeAvgKbits(kDefaultKeyFrameSizeAvgKBits), 38_keyFrameRatio(kDefaultKeyFrameRatio), 39_dropRatio(kDefaultDropRatioAlpha, kDefaultDropRatioMax), 40_enabled(true), 41_max_time_drops(max_time_drops) 42{ 43 Reset(); 44} 45 46void 47FrameDropper::Reset() 48{ 49 _keyFrameRatio.Reset(0.99f); 50 _keyFrameRatio.Apply(1.0f, 1.0f/300.0f); // 1 key frame every 10th second in 30 fps 51 _keyFrameSizeAvgKbits.Reset(0.9f); 52 _keyFrameCount = 0; 53 _accumulator = 0.0f; 54 _accumulatorMax = 150.0f; // assume 300 kb/s and 0.5 s window 55 _targetBitRate = 300.0f; 56 _incoming_frame_rate = 30; 57 _keyFrameSpreadFrames = 0.5f * _incoming_frame_rate; 58 _dropNext = false; 59 _dropRatio.Reset(0.9f); 60 _dropRatio.Apply(0.0f, 0.0f); // Initialize to 0 61 _dropCount = 0; 62 _windowSize = 0.5f; 63 _wasBelowMax = true; 64 _fastMode = false; // start with normal (non-aggressive) mode 65 // Cap for the encoder buffer level/accumulator, in secs. 66 _cap_buffer_size = 3.0f; 67 // Cap on maximum amount of dropped frames between kept frames, in secs. 68 _max_time_drops = 4.0f; 69} 70 71void 72FrameDropper::Enable(bool enable) 73{ 74 _enabled = enable; 75} 76 77void 78FrameDropper::Fill(uint32_t frameSizeBytes, bool deltaFrame) 79{ 80 if (!_enabled) 81 { 82 return; 83 } 84 float frameSizeKbits = 8.0f * static_cast<float>(frameSizeBytes) / 1000.0f; 85 if (!deltaFrame && !_fastMode) // fast mode does not treat key-frames any different 86 { 87 _keyFrameSizeAvgKbits.Apply(1, frameSizeKbits); 88 _keyFrameRatio.Apply(1.0, 1.0); 89 if (frameSizeKbits > _keyFrameSizeAvgKbits.filtered()) 90 { 91 // Remove the average key frame size since we 92 // compensate for key frames when adding delta 93 // frames. 94 frameSizeKbits -= _keyFrameSizeAvgKbits.filtered(); 95 } 96 else 97 { 98 // Shouldn't be negative, so zero is the lower bound. 99 frameSizeKbits = 0; 100 } 101 if (_keyFrameRatio.filtered() > 1e-5 && 102 1 / _keyFrameRatio.filtered() < _keyFrameSpreadFrames) 103 { 104 // We are sending key frames more often than our upper bound for 105 // how much we allow the key frame compensation to be spread 106 // out in time. Therefor we must use the key frame ratio rather 107 // than keyFrameSpreadFrames. 108 _keyFrameCount = 109 static_cast<int32_t>(1 / _keyFrameRatio.filtered() + 0.5); 110 } 111 else 112 { 113 // Compensate for the key frame the following frames 114 _keyFrameCount = static_cast<int32_t>(_keyFrameSpreadFrames + 0.5); 115 } 116 } 117 else 118 { 119 // Decrease the keyFrameRatio 120 _keyFrameRatio.Apply(1.0, 0.0); 121 } 122 // Change the level of the accumulator (bucket) 123 _accumulator += frameSizeKbits; 124 CapAccumulator(); 125} 126 127void 128FrameDropper::Leak(uint32_t inputFrameRate) 129{ 130 if (!_enabled) 131 { 132 return; 133 } 134 if (inputFrameRate < 1) 135 { 136 return; 137 } 138 if (_targetBitRate < 0.0f) 139 { 140 return; 141 } 142 _keyFrameSpreadFrames = 0.5f * inputFrameRate; 143 // T is the expected bits per frame (target). If all frames were the same size, 144 // we would get T bits per frame. Notice that T is also weighted to be able to 145 // force a lower frame rate if wanted. 146 float T = _targetBitRate / inputFrameRate; 147 if (_keyFrameCount > 0) 148 { 149 // Perform the key frame compensation 150 if (_keyFrameRatio.filtered() > 0 && 151 1 / _keyFrameRatio.filtered() < _keyFrameSpreadFrames) 152 { 153 T -= _keyFrameSizeAvgKbits.filtered() * _keyFrameRatio.filtered(); 154 } 155 else 156 { 157 T -= _keyFrameSizeAvgKbits.filtered() / _keyFrameSpreadFrames; 158 } 159 _keyFrameCount--; 160 } 161 _accumulator -= T; 162 if (_accumulator < 0.0f) 163 { 164 _accumulator = 0.0f; 165 } 166 UpdateRatio(); 167} 168 169void 170FrameDropper::UpdateNack(uint32_t nackBytes) 171{ 172 if (!_enabled) 173 { 174 return; 175 } 176 _accumulator += static_cast<float>(nackBytes) * 8.0f / 1000.0f; 177} 178 179void 180FrameDropper::FillBucket(float inKbits, float outKbits) 181{ 182 _accumulator += (inKbits - outKbits); 183} 184 185void 186FrameDropper::UpdateRatio() 187{ 188 if (_accumulator > 1.3f * _accumulatorMax) 189 { 190 // Too far above accumulator max, react faster 191 _dropRatio.UpdateBase(0.8f); 192 } 193 else 194 { 195 // Go back to normal reaction 196 _dropRatio.UpdateBase(0.9f); 197 } 198 if (_accumulator > _accumulatorMax) 199 { 200 // We are above accumulator max, and should ideally 201 // drop a frame. Increase the dropRatio and drop 202 // the frame later. 203 if (_wasBelowMax) 204 { 205 _dropNext = true; 206 } 207 if (_fastMode) 208 { 209 // always drop in aggressive mode 210 _dropNext = true; 211 } 212 213 _dropRatio.Apply(1.0f, 1.0f); 214 _dropRatio.UpdateBase(0.9f); 215 } 216 else 217 { 218 _dropRatio.Apply(1.0f, 0.0f); 219 } 220 _wasBelowMax = _accumulator < _accumulatorMax; 221} 222 223// This function signals when to drop frames to the caller. It makes use of the dropRatio 224// to smooth out the drops over time. 225bool 226FrameDropper::DropFrame() 227{ 228 if (!_enabled) 229 { 230 return false; 231 } 232 if (_dropNext) 233 { 234 _dropNext = false; 235 _dropCount = 0; 236 } 237 238 if (_dropRatio.filtered() >= 0.5f) // Drops per keep 239 { 240 // limit is the number of frames we should drop between each kept frame 241 // to keep our drop ratio. limit is positive in this case. 242 float denom = 1.0f - _dropRatio.filtered(); 243 if (denom < 1e-5) 244 { 245 denom = (float)1e-5; 246 } 247 int32_t limit = static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f); 248 // Put a bound on the max amount of dropped frames between each kept 249 // frame, in terms of frame rate and window size (secs). 250 int max_limit = static_cast<int>(_incoming_frame_rate * 251 _max_time_drops); 252 if (limit > max_limit) { 253 limit = max_limit; 254 } 255 if (_dropCount < 0) 256 { 257 // Reset the _dropCount since it was negative and should be positive. 258 if (_dropRatio.filtered() > 0.4f) 259 { 260 _dropCount = -_dropCount; 261 } 262 else 263 { 264 _dropCount = 0; 265 } 266 } 267 if (_dropCount < limit) 268 { 269 // As long we are below the limit we should drop frames. 270 _dropCount++; 271 return true; 272 } 273 else 274 { 275 // Only when we reset _dropCount a frame should be kept. 276 _dropCount = 0; 277 return false; 278 } 279 } 280 else if (_dropRatio.filtered() > 0.0f && 281 _dropRatio.filtered() < 0.5f) // Keeps per drop 282 { 283 // limit is the number of frames we should keep between each drop 284 // in order to keep the drop ratio. limit is negative in this case, 285 // and the _dropCount is also negative. 286 float denom = _dropRatio.filtered(); 287 if (denom < 1e-5) 288 { 289 denom = (float)1e-5; 290 } 291 int32_t limit = -static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f); 292 if (_dropCount > 0) 293 { 294 // Reset the _dropCount since we have a positive 295 // _dropCount, and it should be negative. 296 if (_dropRatio.filtered() < 0.6f) 297 { 298 _dropCount = -_dropCount; 299 } 300 else 301 { 302 _dropCount = 0; 303 } 304 } 305 if (_dropCount > limit) 306 { 307 if (_dropCount == 0) 308 { 309 // Drop frames when we reset _dropCount. 310 _dropCount--; 311 return true; 312 } 313 else 314 { 315 // Keep frames as long as we haven't reached limit. 316 _dropCount--; 317 return false; 318 } 319 } 320 else 321 { 322 _dropCount = 0; 323 return false; 324 } 325 } 326 _dropCount = 0; 327 return false; 328 329 // A simpler version, unfiltered and quicker 330 //bool dropNext = _dropNext; 331 //_dropNext = false; 332 //return dropNext; 333} 334 335void 336FrameDropper::SetRates(float bitRate, float incoming_frame_rate) 337{ 338 // Bit rate of -1 means infinite bandwidth. 339 _accumulatorMax = bitRate * _windowSize; // bitRate * windowSize (in seconds) 340 if (_targetBitRate > 0.0f && bitRate < _targetBitRate && _accumulator > _accumulatorMax) 341 { 342 // Rescale the accumulator level if the accumulator max decreases 343 _accumulator = bitRate / _targetBitRate * _accumulator; 344 } 345 _targetBitRate = bitRate; 346 CapAccumulator(); 347 _incoming_frame_rate = incoming_frame_rate; 348} 349 350float 351FrameDropper::ActualFrameRate(uint32_t inputFrameRate) const 352{ 353 if (!_enabled) 354 { 355 return static_cast<float>(inputFrameRate); 356 } 357 return inputFrameRate * (1.0f - _dropRatio.filtered()); 358} 359 360// Put a cap on the accumulator, i.e., don't let it grow beyond some level. 361// This is a temporary fix for screencasting where very large frames from 362// encoder will cause very slow response (too many frame drops). 363void FrameDropper::CapAccumulator() { 364 float max_accumulator = _targetBitRate * _cap_buffer_size; 365 if (_accumulator > max_accumulator) { 366 _accumulator = max_accumulator; 367 } 368} 369 370} 371