1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#ifndef WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
12#define WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
13
14#include <math.h>
15#include <stdlib.h>
16
17#include "webrtc/base/exp_filter.h"
18#include "webrtc/base/scoped_ptr.h"
19#include "webrtc/modules/video_coding/internal_defines.h"
20#include "webrtc/modules/video_coding/qm_select.h"
21#include "webrtc/system_wrappers/include/trace.h"
22#include "webrtc/typedefs.h"
23
24namespace webrtc {
25namespace media_optimization {
26
27// Number of time periods used for (max) window filter for packet loss
28// TODO(marpan): set reasonable window size for filtered packet loss,
29// adjustment should be based on logged/real data of loss stats/correlation.
30enum { kLossPrHistorySize = 10 };
31
32// 1000 ms, total filter length is (kLossPrHistorySize * 1000) ms
33enum { kLossPrShortFilterWinMs = 1000 };
34
35// The type of filter used on the received packet loss reports.
36enum FilterPacketLossMode {
37  kNoFilter,   // No filtering on received loss.
38  kAvgFilter,  // Recursive average filter.
39  kMaxFilter   // Max-window filter, over the time interval of:
40               // (kLossPrHistorySize * kLossPrShortFilterWinMs) ms.
41};
42
43// Thresholds for hybrid NACK/FEC
44// common to media optimization and the jitter buffer.
45const int64_t kLowRttNackMs = 20;
46
47struct VCMProtectionParameters {
48  VCMProtectionParameters()
49      : rtt(0),
50        lossPr(0.0f),
51        bitRate(0.0f),
52        packetsPerFrame(0.0f),
53        packetsPerFrameKey(0.0f),
54        frameRate(0.0f),
55        keyFrameSize(0.0f),
56        fecRateDelta(0),
57        fecRateKey(0),
58        codecWidth(0),
59        codecHeight(0),
60        numLayers(1) {}
61
62  int64_t rtt;
63  float lossPr;
64  float bitRate;
65  float packetsPerFrame;
66  float packetsPerFrameKey;
67  float frameRate;
68  float keyFrameSize;
69  uint8_t fecRateDelta;
70  uint8_t fecRateKey;
71  uint16_t codecWidth;
72  uint16_t codecHeight;
73  int numLayers;
74};
75
76/******************************/
77/* VCMProtectionMethod class  */
78/******************************/
79
80enum VCMProtectionMethodEnum { kNack, kFec, kNackFec, kNone };
81
82class VCMLossProbabilitySample {
83 public:
84  VCMLossProbabilitySample() : lossPr255(0), timeMs(-1) {}
85
86  uint8_t lossPr255;
87  int64_t timeMs;
88};
89
90class VCMProtectionMethod {
91 public:
92  VCMProtectionMethod();
93  virtual ~VCMProtectionMethod();
94
95  // Updates the efficiency of the method using the parameters provided
96  //
97  // Input:
98  //         - parameters         : Parameters used to calculate efficiency
99  //
100  // Return value                 : True if this method is recommended in
101  //                                the given conditions.
102  virtual bool UpdateParameters(const VCMProtectionParameters* parameters) = 0;
103
104  // Returns the protection type
105  //
106  // Return value                 : The protection type
107  enum VCMProtectionMethodEnum Type() const { return _type; }
108
109  // Returns the effective packet loss for ER, required by this protection
110  // method
111  //
112  // Return value                 : Required effective packet loss
113  virtual uint8_t RequiredPacketLossER() { return _effectivePacketLoss; }
114
115  // Extracts the FEC protection factor for Key frame, required by this
116  // protection method
117  //
118  // Return value                 : Required protectionFactor for Key frame
119  virtual uint8_t RequiredProtectionFactorK() { return _protectionFactorK; }
120
121  // Extracts the FEC protection factor for Delta frame, required by this
122  // protection method
123  //
124  // Return value                 : Required protectionFactor for delta frame
125  virtual uint8_t RequiredProtectionFactorD() { return _protectionFactorD; }
126
127  // Extracts whether the FEC Unequal protection (UEP) is used for Key frame.
128  //
129  // Return value                 : Required Unequal protection on/off state.
130  virtual bool RequiredUepProtectionK() { return _useUepProtectionK; }
131
132  // Extracts whether the the FEC Unequal protection (UEP) is used for Delta
133  // frame.
134  //
135  // Return value                 : Required Unequal protection on/off state.
136  virtual bool RequiredUepProtectionD() { return _useUepProtectionD; }
137
138  virtual int MaxFramesFec() const { return 1; }
139
140  // Updates content metrics
141  void UpdateContentMetrics(const VideoContentMetrics* contentMetrics);
142
143 protected:
144  uint8_t _effectivePacketLoss;
145  uint8_t _protectionFactorK;
146  uint8_t _protectionFactorD;
147  // Estimation of residual loss after the FEC
148  float _scaleProtKey;
149  int32_t _maxPayloadSize;
150
151  VCMQmRobustness* _qmRobustness;
152  bool _useUepProtectionK;
153  bool _useUepProtectionD;
154  float _corrFecCost;
155  enum VCMProtectionMethodEnum _type;
156};
157
158class VCMNackMethod : public VCMProtectionMethod {
159 public:
160  VCMNackMethod();
161  virtual ~VCMNackMethod();
162  virtual bool UpdateParameters(const VCMProtectionParameters* parameters);
163  // Get the effective packet loss
164  bool EffectivePacketLoss(const VCMProtectionParameters* parameter);
165};
166
167class VCMFecMethod : public VCMProtectionMethod {
168 public:
169  VCMFecMethod();
170  virtual ~VCMFecMethod();
171  virtual bool UpdateParameters(const VCMProtectionParameters* parameters);
172  // Get the effective packet loss for ER
173  bool EffectivePacketLoss(const VCMProtectionParameters* parameters);
174  // Get the FEC protection factors
175  bool ProtectionFactor(const VCMProtectionParameters* parameters);
176  // Get the boost for key frame protection
177  uint8_t BoostCodeRateKey(uint8_t packetFrameDelta,
178                           uint8_t packetFrameKey) const;
179  // Convert the rates: defined relative to total# packets or source# packets
180  uint8_t ConvertFECRate(uint8_t codeRate) const;
181  // Get the average effective recovery from FEC: for random loss model
182  float AvgRecoveryFEC(const VCMProtectionParameters* parameters) const;
183  // Update FEC with protectionFactorD
184  void UpdateProtectionFactorD(uint8_t protectionFactorD);
185  // Update FEC with protectionFactorK
186  void UpdateProtectionFactorK(uint8_t protectionFactorK);
187  // Compute the bits per frame. Account for temporal layers when applicable.
188  int BitsPerFrame(const VCMProtectionParameters* parameters);
189
190 protected:
191  enum { kUpperLimitFramesFec = 6 };
192  // Thresholds values for the bytes/frame and round trip time, below which we
193  // may turn off FEC, depending on |_numLayers| and |_maxFramesFec|.
194  // Max bytes/frame for VGA, corresponds to ~140k at 25fps.
195  enum { kMaxBytesPerFrameForFec = 700 };
196  // Max bytes/frame for CIF and lower: corresponds to ~80k at 25fps.
197  enum { kMaxBytesPerFrameForFecLow = 400 };
198  // Max bytes/frame for frame size larger than VGA, ~200k at 25fps.
199  enum { kMaxBytesPerFrameForFecHigh = 1000 };
200};
201
202class VCMNackFecMethod : public VCMFecMethod {
203 public:
204  VCMNackFecMethod(int64_t lowRttNackThresholdMs,
205                   int64_t highRttNackThresholdMs);
206  virtual ~VCMNackFecMethod();
207  virtual bool UpdateParameters(const VCMProtectionParameters* parameters);
208  // Get the effective packet loss for ER
209  bool EffectivePacketLoss(const VCMProtectionParameters* parameters);
210  // Get the protection factors
211  bool ProtectionFactor(const VCMProtectionParameters* parameters);
212  // Get the max number of frames the FEC is allowed to be based on.
213  int MaxFramesFec() const;
214  // Turn off the FEC based on low bitrate and other factors.
215  bool BitRateTooLowForFec(const VCMProtectionParameters* parameters);
216
217 private:
218  int ComputeMaxFramesFec(const VCMProtectionParameters* parameters);
219
220  int64_t _lowRttNackMs;
221  int64_t _highRttNackMs;
222  int _maxFramesFec;
223};
224
225class VCMLossProtectionLogic {
226 public:
227  explicit VCMLossProtectionLogic(int64_t nowMs);
228  ~VCMLossProtectionLogic();
229
230  // Set the protection method to be used
231  //
232  // Input:
233  //        - newMethodType    : New requested protection method type. If one
234  //                           is already set, it will be deleted and replaced
235  void SetMethod(VCMProtectionMethodEnum newMethodType);
236
237  // Update the round-trip time
238  //
239  // Input:
240  //          - rtt           : Round-trip time in seconds.
241  void UpdateRtt(int64_t rtt);
242
243  // Update the filtered packet loss.
244  //
245  // Input:
246  //          - packetLossEnc :  The reported packet loss filtered
247  //                             (max window or average)
248  void UpdateFilteredLossPr(uint8_t packetLossEnc);
249
250  // Update the current target bit rate.
251  //
252  // Input:
253  //          - bitRate          : The current target bit rate in kbits/s
254  void UpdateBitRate(float bitRate);
255
256  // Update the number of packets per frame estimate, for delta frames
257  //
258  // Input:
259  //          - nPackets         : Number of packets in the latest sent frame.
260  void UpdatePacketsPerFrame(float nPackets, int64_t nowMs);
261
262  // Update the number of packets per frame estimate, for key frames
263  //
264  // Input:
265  //          - nPackets         : umber of packets in the latest sent frame.
266  void UpdatePacketsPerFrameKey(float nPackets, int64_t nowMs);
267
268  // Update the keyFrameSize estimate
269  //
270  // Input:
271  //          - keyFrameSize     : The size of the latest sent key frame.
272  void UpdateKeyFrameSize(float keyFrameSize);
273
274  // Update the frame rate
275  //
276  // Input:
277  //          - frameRate        : The current target frame rate.
278  void UpdateFrameRate(float frameRate) { _frameRate = frameRate; }
279
280  // Update the frame size
281  //
282  // Input:
283  //          - width        : The codec frame width.
284  //          - height       : The codec frame height.
285  void UpdateFrameSize(uint16_t width, uint16_t height);
286
287  // Update the number of active layers
288  //
289  // Input:
290  //          - numLayers    : Number of layers used.
291  void UpdateNumLayers(int numLayers);
292
293  // The amount of packet loss to cover for with FEC.
294  //
295  // Input:
296  //          - fecRateKey      : Packet loss to cover for with FEC when
297  //                              sending key frames.
298  //          - fecRateDelta    : Packet loss to cover for with FEC when
299  //                              sending delta frames.
300  void UpdateFECRates(uint8_t fecRateKey, uint8_t fecRateDelta) {
301    _fecRateKey = fecRateKey;
302    _fecRateDelta = fecRateDelta;
303  }
304
305  // Update the protection methods with the current VCMProtectionParameters
306  // and set the requested protection settings.
307  // Return value     : Returns true on update
308  bool UpdateMethod();
309
310  // Returns the method currently selected.
311  //
312  // Return value                 : The protection method currently selected.
313  VCMProtectionMethod* SelectedMethod() const;
314
315  // Return the protection type of the currently selected method
316  VCMProtectionMethodEnum SelectedType() const;
317
318  // Updates the filtered loss for the average and max window packet loss,
319  // and returns the filtered loss probability in the interval [0, 255].
320  // The returned filtered loss value depends on the parameter |filter_mode|.
321  // The input parameter |lossPr255| is the received packet loss.
322
323  // Return value                 : The filtered loss probability
324  uint8_t FilteredLoss(int64_t nowMs,
325                       FilterPacketLossMode filter_mode,
326                       uint8_t lossPr255);
327
328  void Reset(int64_t nowMs);
329
330  void Release();
331
332 private:
333  // Sets the available loss protection methods.
334  void UpdateMaxLossHistory(uint8_t lossPr255, int64_t now);
335  uint8_t MaxFilteredLossPr(int64_t nowMs) const;
336  rtc::scoped_ptr<VCMProtectionMethod> _selectedMethod;
337  VCMProtectionParameters _currentParameters;
338  int64_t _rtt;
339  float _lossPr;
340  float _bitRate;
341  float _frameRate;
342  float _keyFrameSize;
343  uint8_t _fecRateKey;
344  uint8_t _fecRateDelta;
345  int64_t _lastPrUpdateT;
346  int64_t _lastPacketPerFrameUpdateT;
347  int64_t _lastPacketPerFrameUpdateTKey;
348  rtc::ExpFilter _lossPr255;
349  VCMLossProbabilitySample _lossPrHistory[kLossPrHistorySize];
350  uint8_t _shortMaxLossPr255;
351  rtc::ExpFilter _packetsPerFrame;
352  rtc::ExpFilter _packetsPerFrameKey;
353  uint16_t _codecWidth;
354  uint16_t _codecHeight;
355  int _numLayers;
356};
357
358}  // namespace media_optimization
359}  // namespace webrtc
360
361#endif  // WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
362