1/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 2* 3* Use of this source code is governed by a BSD-style license 4* that can be found in the LICENSE file in the root of the source 5* tree. An additional intellectual property rights grant can be found 6* in the file PATENTS. All contributing project authors may 7* be found in the AUTHORS file in the root of the source tree. 8*/ 9 10#include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" 11 12#include <stdlib.h> 13 14#include <algorithm> 15 16#include "webrtc/base/checks.h" 17#include "vpx/vpx_encoder.h" 18#include "vpx/vp8cx.h" 19#include "webrtc/modules/video_coding/include/video_codec_interface.h" 20 21namespace webrtc { 22 23static const int kOneSecond90Khz = 90000; 24static const int kMinTimeBetweenSyncs = kOneSecond90Khz * 5; 25static const int kMaxTimeBetweenSyncs = kOneSecond90Khz * 10; 26static const int kQpDeltaThresholdForSync = 8; 27 28const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5; 29const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0; 30 31// Since this is TL0 we only allow updating and predicting from the LAST 32// reference frame. 33const int ScreenshareLayers::kTl0Flags = 34 VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF | 35 VP8_EFLAG_NO_REF_ARF; 36 37// Allow predicting from both TL0 and TL1. 38const int ScreenshareLayers::kTl1Flags = 39 VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST; 40 41// Allow predicting from only TL0 to allow participants to switch to the high 42// bitrate stream. This means predicting only from the LAST reference frame, but 43// only updating GF to not corrupt TL0. 44const int ScreenshareLayers::kTl1SyncFlags = 45 VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | 46 VP8_EFLAG_NO_UPD_LAST; 47 48ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, 49 uint8_t initial_tl0_pic_idx) 50 : number_of_temporal_layers_(num_temporal_layers), 51 last_base_layer_sync_(false), 52 tl0_pic_idx_(initial_tl0_pic_idx), 53 active_layer_(-1), 54 last_timestamp_(-1), 55 last_sync_timestamp_(-1), 56 min_qp_(-1), 57 max_qp_(-1), 58 max_debt_bytes_(0), 59 frame_rate_(-1) { 60 assert(num_temporal_layers > 0); 61 assert(num_temporal_layers <= 2); 62} 63 64int ScreenshareLayers::CurrentLayerId() const { 65 // Codec does not use temporal layers for screenshare. 66 return 0; 67} 68 69int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { 70 if (number_of_temporal_layers_ <= 1) { 71 // No flags needed for 1 layer screenshare. 72 return 0; 73 } 74 75 int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); 76 int flags = 0; 77 78 if (active_layer_ == -1 || 79 layers_[active_layer_].state != TemporalLayer::State::kDropped) { 80 if (layers_[0].debt_bytes_ > max_debt_bytes_) { 81 // Must drop TL0, encode TL1 instead. 82 if (layers_[1].debt_bytes_ > max_debt_bytes_) { 83 // Must drop both TL0 and TL1. 84 active_layer_ = -1; 85 } else { 86 active_layer_ = 1; 87 } 88 } else { 89 active_layer_ = 0; 90 } 91 } 92 93 switch (active_layer_) { 94 case 0: 95 flags = kTl0Flags; 96 break; 97 case 1: 98 if (TimeToSync(unwrapped_timestamp)) { 99 last_sync_timestamp_ = unwrapped_timestamp; 100 flags = kTl1SyncFlags; 101 } else { 102 flags = kTl1Flags; 103 } 104 break; 105 case -1: 106 flags = -1; 107 break; 108 default: 109 flags = -1; 110 RTC_NOTREACHED(); 111 } 112 113 // Make sure both frame droppers leak out bits. 114 int64_t ts_diff; 115 if (last_timestamp_ == -1) { 116 ts_diff = kOneSecond90Khz / (frame_rate_ <= 0 ? 5 : frame_rate_); 117 } else { 118 ts_diff = unwrapped_timestamp - last_timestamp_; 119 } 120 121 layers_[0].UpdateDebt(ts_diff / 90); 122 layers_[1].UpdateDebt(ts_diff / 90); 123 last_timestamp_ = timestamp; 124 return flags; 125} 126 127bool ScreenshareLayers::ConfigureBitrates(int bitrate_kbps, 128 int max_bitrate_kbps, 129 int framerate, 130 vpx_codec_enc_cfg_t* cfg) { 131 layers_[0].target_rate_kbps_ = bitrate_kbps; 132 layers_[1].target_rate_kbps_ = max_bitrate_kbps; 133 134 int target_bitrate_kbps = bitrate_kbps; 135 136 if (cfg != nullptr) { 137 if (number_of_temporal_layers_ > 1) { 138 // Calculate a codec target bitrate. This may be higher than TL0, gaining 139 // quality at the expense of frame rate at TL0. Constraints: 140 // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction. 141 // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate. 142 target_bitrate_kbps = 143 std::min(bitrate_kbps * kMaxTL0FpsReduction, 144 max_bitrate_kbps / kAcceptableTargetOvershoot); 145 146 cfg->rc_target_bitrate = std::max(bitrate_kbps, target_bitrate_kbps); 147 } 148 149 // Don't reconfigure qp limits during quality boost frames. 150 if (layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) { 151 min_qp_ = cfg->rc_min_quantizer; 152 max_qp_ = cfg->rc_max_quantizer; 153 // After a dropped frame, a frame with max qp will be encoded and the 154 // quality will then ramp up from there. To boost the speed of recovery, 155 // encode the next frame with lower max qp. TL0 is the most important to 156 // improve since the errors in this layer will propagate to TL1. 157 // Currently, reduce max qp by 20% for TL0 and 15% for TL1. 158 layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100); 159 layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100); 160 } 161 } 162 163 int avg_frame_size = (target_bitrate_kbps * 1000) / (8 * framerate); 164 max_debt_bytes_ = 4 * avg_frame_size; 165 166 return true; 167} 168 169void ScreenshareLayers::FrameEncoded(unsigned int size, 170 uint32_t timestamp, 171 int qp) { 172 if (size == 0) { 173 layers_[active_layer_].state = TemporalLayer::State::kDropped; 174 return; 175 } 176 177 if (layers_[active_layer_].state == TemporalLayer::State::kDropped) { 178 layers_[active_layer_].state = TemporalLayer::State::kQualityBoost; 179 } 180 181 if (qp != -1) 182 layers_[active_layer_].last_qp = qp; 183 184 if (active_layer_ == 0) { 185 layers_[0].debt_bytes_ += size; 186 layers_[1].debt_bytes_ += size; 187 } else if (active_layer_ == 1) { 188 layers_[1].debt_bytes_ += size; 189 } 190} 191 192void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync, 193 CodecSpecificInfoVP8* vp8_info, 194 uint32_t timestamp) { 195 int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); 196 if (number_of_temporal_layers_ == 1) { 197 vp8_info->temporalIdx = kNoTemporalIdx; 198 vp8_info->layerSync = false; 199 vp8_info->tl0PicIdx = kNoTl0PicIdx; 200 } else { 201 vp8_info->temporalIdx = active_layer_; 202 if (base_layer_sync) { 203 vp8_info->temporalIdx = 0; 204 last_sync_timestamp_ = unwrapped_timestamp; 205 } else if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) { 206 // Regardless of pattern the frame after a base layer sync will always 207 // be a layer sync. 208 last_sync_timestamp_ = unwrapped_timestamp; 209 } 210 vp8_info->layerSync = last_sync_timestamp_ != -1 && 211 last_sync_timestamp_ == unwrapped_timestamp; 212 if (vp8_info->temporalIdx == 0) { 213 tl0_pic_idx_++; 214 } 215 last_base_layer_sync_ = base_layer_sync; 216 vp8_info->tl0PicIdx = tl0_pic_idx_; 217 } 218} 219 220bool ScreenshareLayers::TimeToSync(int64_t timestamp) const { 221 if (active_layer_ != 1) { 222 RTC_NOTREACHED(); 223 return false; 224 } 225 RTC_DCHECK_NE(-1, layers_[0].last_qp); 226 if (layers_[1].last_qp == -1) { 227 // First frame in TL1 should only depend on TL0 since there are no 228 // previous frames in TL1. 229 return true; 230 } 231 232 RTC_DCHECK_NE(-1, last_sync_timestamp_); 233 int64_t timestamp_diff = timestamp - last_sync_timestamp_; 234 if (timestamp_diff > kMaxTimeBetweenSyncs) { 235 // After a certain time, force a sync frame. 236 return true; 237 } else if (timestamp_diff < kMinTimeBetweenSyncs) { 238 // If too soon from previous sync frame, don't issue a new one. 239 return false; 240 } 241 // Issue a sync frame if difference in quality between TL0 and TL1 isn't too 242 // large. 243 if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync) 244 return true; 245 return false; 246} 247 248bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { 249 if (max_qp_ == -1 || number_of_temporal_layers_ <= 1) 250 return false; 251 252 // If layer is in the quality boost state (following a dropped frame), update 253 // the configuration with the adjusted (lower) qp and set the state back to 254 // normal. 255 unsigned int adjusted_max_qp; 256 if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost && 257 layers_[active_layer_].enhanced_max_qp != -1) { 258 adjusted_max_qp = layers_[active_layer_].enhanced_max_qp; 259 layers_[active_layer_].state = TemporalLayer::State::kNormal; 260 } else { 261 if (max_qp_ == -1) 262 return false; 263 adjusted_max_qp = max_qp_; // Set the normal max qp. 264 } 265 266 if (adjusted_max_qp == cfg->rc_max_quantizer) 267 return false; 268 269 cfg->rc_max_quantizer = adjusted_max_qp; 270 return true; 271} 272 273void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) { 274 uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8; 275 if (debt_reduction_bytes >= debt_bytes_) { 276 debt_bytes_ = 0; 277 } else { 278 debt_bytes_ -= debt_reduction_bytes; 279 } 280} 281 282} // namespace webrtc 283