1/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
2*
3*  Use of this source code is governed by a BSD-style license
4*  that can be found in the LICENSE file in the root of the source
5*  tree. An additional intellectual property rights grant can be found
6*  in the file PATENTS.  All contributing project authors may
7*  be found in the AUTHORS file in the root of the source tree.
8*/
9
10#include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
11
12#include <stdlib.h>
13
14#include <algorithm>
15
16#include "webrtc/base/checks.h"
17#include "vpx/vpx_encoder.h"
18#include "vpx/vp8cx.h"
19#include "webrtc/modules/video_coding/include/video_codec_interface.h"
20
21namespace webrtc {
22
23static const int kOneSecond90Khz = 90000;
24static const int kMinTimeBetweenSyncs = kOneSecond90Khz * 5;
25static const int kMaxTimeBetweenSyncs = kOneSecond90Khz * 10;
26static const int kQpDeltaThresholdForSync = 8;
27
28const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
29const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
30
31// Since this is TL0 we only allow updating and predicting from the LAST
32// reference frame.
33const int ScreenshareLayers::kTl0Flags =
34    VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF |
35    VP8_EFLAG_NO_REF_ARF;
36
37// Allow predicting from both TL0 and TL1.
38const int ScreenshareLayers::kTl1Flags =
39    VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
40
41// Allow predicting from only TL0 to allow participants to switch to the high
42// bitrate stream. This means predicting only from the LAST reference frame, but
43// only updating GF to not corrupt TL0.
44const int ScreenshareLayers::kTl1SyncFlags =
45    VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF |
46    VP8_EFLAG_NO_UPD_LAST;
47
48ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
49                                     uint8_t initial_tl0_pic_idx)
50    : number_of_temporal_layers_(num_temporal_layers),
51      last_base_layer_sync_(false),
52      tl0_pic_idx_(initial_tl0_pic_idx),
53      active_layer_(-1),
54      last_timestamp_(-1),
55      last_sync_timestamp_(-1),
56      min_qp_(-1),
57      max_qp_(-1),
58      max_debt_bytes_(0),
59      frame_rate_(-1) {
60  assert(num_temporal_layers > 0);
61  assert(num_temporal_layers <= 2);
62}
63
64int ScreenshareLayers::CurrentLayerId() const {
65  // Codec does not use temporal layers for screenshare.
66  return 0;
67}
68
69int ScreenshareLayers::EncodeFlags(uint32_t timestamp) {
70  if (number_of_temporal_layers_ <= 1) {
71    // No flags needed for 1 layer screenshare.
72    return 0;
73  }
74
75  int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
76  int flags = 0;
77
78  if (active_layer_ == -1 ||
79      layers_[active_layer_].state != TemporalLayer::State::kDropped) {
80    if (layers_[0].debt_bytes_ > max_debt_bytes_) {
81      // Must drop TL0, encode TL1 instead.
82      if (layers_[1].debt_bytes_ > max_debt_bytes_) {
83        // Must drop both TL0 and TL1.
84        active_layer_ = -1;
85      } else {
86        active_layer_ = 1;
87      }
88    } else {
89      active_layer_ = 0;
90    }
91  }
92
93  switch (active_layer_) {
94    case 0:
95      flags = kTl0Flags;
96      break;
97    case 1:
98      if (TimeToSync(unwrapped_timestamp)) {
99        last_sync_timestamp_ = unwrapped_timestamp;
100        flags = kTl1SyncFlags;
101      } else {
102        flags = kTl1Flags;
103      }
104      break;
105    case -1:
106      flags = -1;
107      break;
108    default:
109      flags = -1;
110      RTC_NOTREACHED();
111  }
112
113  // Make sure both frame droppers leak out bits.
114  int64_t ts_diff;
115  if (last_timestamp_ == -1) {
116    ts_diff = kOneSecond90Khz / (frame_rate_ <= 0 ? 5 : frame_rate_);
117  } else {
118    ts_diff = unwrapped_timestamp - last_timestamp_;
119  }
120
121  layers_[0].UpdateDebt(ts_diff / 90);
122  layers_[1].UpdateDebt(ts_diff / 90);
123  last_timestamp_ = timestamp;
124  return flags;
125}
126
127bool ScreenshareLayers::ConfigureBitrates(int bitrate_kbps,
128                                          int max_bitrate_kbps,
129                                          int framerate,
130                                          vpx_codec_enc_cfg_t* cfg) {
131  layers_[0].target_rate_kbps_ = bitrate_kbps;
132  layers_[1].target_rate_kbps_ = max_bitrate_kbps;
133
134  int target_bitrate_kbps = bitrate_kbps;
135
136  if (cfg != nullptr) {
137    if (number_of_temporal_layers_ > 1) {
138      // Calculate a codec target bitrate. This may be higher than TL0, gaining
139      // quality at the expense of frame rate at TL0. Constraints:
140      // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction.
141      // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate.
142      target_bitrate_kbps =
143          std::min(bitrate_kbps * kMaxTL0FpsReduction,
144                   max_bitrate_kbps / kAcceptableTargetOvershoot);
145
146      cfg->rc_target_bitrate = std::max(bitrate_kbps, target_bitrate_kbps);
147    }
148
149    // Don't reconfigure qp limits during quality boost frames.
150    if (layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) {
151      min_qp_ = cfg->rc_min_quantizer;
152      max_qp_ = cfg->rc_max_quantizer;
153      // After a dropped frame, a frame with max qp will be encoded and the
154      // quality will then ramp up from there. To boost the speed of recovery,
155      // encode the next frame with lower max qp. TL0 is the most important to
156      // improve since the errors in this layer will propagate to TL1.
157      // Currently, reduce max qp by 20% for TL0 and 15% for TL1.
158      layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100);
159      layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100);
160    }
161  }
162
163  int avg_frame_size = (target_bitrate_kbps * 1000) / (8 * framerate);
164  max_debt_bytes_ = 4 * avg_frame_size;
165
166  return true;
167}
168
169void ScreenshareLayers::FrameEncoded(unsigned int size,
170                                     uint32_t timestamp,
171                                     int qp) {
172  if (size == 0) {
173    layers_[active_layer_].state = TemporalLayer::State::kDropped;
174    return;
175  }
176
177  if (layers_[active_layer_].state == TemporalLayer::State::kDropped) {
178    layers_[active_layer_].state = TemporalLayer::State::kQualityBoost;
179  }
180
181  if (qp != -1)
182    layers_[active_layer_].last_qp = qp;
183
184  if (active_layer_ == 0) {
185    layers_[0].debt_bytes_ += size;
186    layers_[1].debt_bytes_ += size;
187  } else if (active_layer_ == 1) {
188    layers_[1].debt_bytes_ += size;
189  }
190}
191
192void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync,
193                                              CodecSpecificInfoVP8* vp8_info,
194                                              uint32_t timestamp) {
195  int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
196  if (number_of_temporal_layers_ == 1) {
197    vp8_info->temporalIdx = kNoTemporalIdx;
198    vp8_info->layerSync = false;
199    vp8_info->tl0PicIdx = kNoTl0PicIdx;
200  } else {
201    vp8_info->temporalIdx = active_layer_;
202    if (base_layer_sync) {
203      vp8_info->temporalIdx = 0;
204      last_sync_timestamp_ = unwrapped_timestamp;
205    } else if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) {
206      // Regardless of pattern the frame after a base layer sync will always
207      // be a layer sync.
208      last_sync_timestamp_ = unwrapped_timestamp;
209    }
210    vp8_info->layerSync = last_sync_timestamp_ != -1 &&
211                          last_sync_timestamp_ == unwrapped_timestamp;
212    if (vp8_info->temporalIdx == 0) {
213      tl0_pic_idx_++;
214    }
215    last_base_layer_sync_ = base_layer_sync;
216    vp8_info->tl0PicIdx = tl0_pic_idx_;
217  }
218}
219
220bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
221  if (active_layer_ != 1) {
222    RTC_NOTREACHED();
223    return false;
224  }
225  RTC_DCHECK_NE(-1, layers_[0].last_qp);
226  if (layers_[1].last_qp == -1) {
227    // First frame in TL1 should only depend on TL0 since there are no
228    // previous frames in TL1.
229    return true;
230  }
231
232  RTC_DCHECK_NE(-1, last_sync_timestamp_);
233  int64_t timestamp_diff = timestamp - last_sync_timestamp_;
234  if (timestamp_diff > kMaxTimeBetweenSyncs) {
235    // After a certain time, force a sync frame.
236    return true;
237  } else if (timestamp_diff < kMinTimeBetweenSyncs) {
238    // If too soon from previous sync frame, don't issue a new one.
239    return false;
240  }
241  // Issue a sync frame if difference in quality between TL0 and TL1 isn't too
242  // large.
243  if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync)
244    return true;
245  return false;
246}
247
248bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) {
249  if (max_qp_ == -1 || number_of_temporal_layers_ <= 1)
250    return false;
251
252  // If layer is in the quality boost state (following a dropped frame), update
253  // the configuration with the adjusted (lower) qp and set the state back to
254  // normal.
255  unsigned int adjusted_max_qp;
256  if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost &&
257      layers_[active_layer_].enhanced_max_qp != -1) {
258    adjusted_max_qp = layers_[active_layer_].enhanced_max_qp;
259    layers_[active_layer_].state = TemporalLayer::State::kNormal;
260  } else {
261    if (max_qp_ == -1)
262      return false;
263    adjusted_max_qp = max_qp_;  // Set the normal max qp.
264  }
265
266  if (adjusted_max_qp == cfg->rc_max_quantizer)
267    return false;
268
269  cfg->rc_max_quantizer = adjusted_max_qp;
270  return true;
271}
272
273void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) {
274  uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8;
275  if (debt_reduction_bytes >= debt_bytes_) {
276    debt_bytes_ = 0;
277  } else {
278    debt_bytes_ -= debt_reduction_bytes;
279  }
280}
281
282}  // namespace webrtc
283