1b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org/* 2b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * 4b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * Use of this source code is governed by a BSD-style license 5b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * that can be found in the LICENSE file in the root of the source 6b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * tree. An additional intellectual property rights grant can be found 7b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * in the file PATENTS. All contributing project authors may 8b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * be found in the AUTHORS file in the root of the source tree. 9b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org */ 10b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 11a557f436b9d694d5a0a045e0295e1794f2df48eapbos@webrtc.org#include "webrtc/modules/video_coding/main/source/qm_select.h" 12b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 13b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org#include <math.h> 14b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 15a557f436b9d694d5a0a045e0295e1794f2df48eapbos@webrtc.org#include "webrtc/modules/interface/module_common_types.h" 16a557f436b9d694d5a0a045e0295e1794f2df48eapbos@webrtc.org#include "webrtc/modules/video_coding/main/interface/video_coding_defines.h" 17a557f436b9d694d5a0a045e0295e1794f2df48eapbos@webrtc.org#include "webrtc/modules/video_coding/main/source/internal_defines.h" 18a557f436b9d694d5a0a045e0295e1794f2df48eapbos@webrtc.org#include "webrtc/modules/video_coding/main/source/qm_select_data.h" 19a557f436b9d694d5a0a045e0295e1794f2df48eapbos@webrtc.org#include "webrtc/system_wrappers/interface/trace.h" 20b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 21b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgnamespace webrtc { 22b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 23b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// QM-METHOD class 24b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 25b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgVCMQmMethod::VCMQmMethod() 26b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org : content_metrics_(NULL), 27b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org width_(0), 28b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org height_(0), 29b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org user_frame_rate_(0.0f), 30b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org native_width_(0), 31b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org native_height_(0), 32b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org native_frame_rate_(0.0f), 33b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org image_type_(kVGA), 34b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org framerate_level_(kFrameRateHigh), 35b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org init_(false) { 36b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ResetQM(); 37b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 38b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 39b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgVCMQmMethod::~VCMQmMethod() { 40b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 41b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 42b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmMethod::ResetQM() { 43b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org aspect_ratio_ = 1.0f; 44b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org motion_.Reset(); 45b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_.Reset(); 46b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org content_class_ = 0; 47b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 48b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 49b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orguint8_t VCMQmMethod::ComputeContentClass() { 50b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ComputeMotionNFD(); 51b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ComputeSpatial(); 52b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return content_class_ = 3 * motion_.level + spatial_.level; 53b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 54b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 55b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmMethod::UpdateContent(const VideoContentMetrics* contentMetrics) { 56b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org content_metrics_ = contentMetrics; 57b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 58b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 59b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmMethod::ComputeMotionNFD() { 60b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (content_metrics_) { 61b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org motion_.value = content_metrics_->motion_magnitude; 62b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 63b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Determine motion level. 64b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (motion_.value < kLowMotionNfd) { 65b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org motion_.level = kLow; 66b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (motion_.value > kHighMotionNfd) { 67b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org motion_.level = kHigh; 68b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else { 69b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org motion_.level = kDefault; 70b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 71b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 72b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 73b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmMethod::ComputeSpatial() { 74b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float spatial_err = 0.0; 75b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float spatial_err_h = 0.0; 76b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float spatial_err_v = 0.0; 77b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (content_metrics_) { 78b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_err = content_metrics_->spatial_pred_err; 79b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_err_h = content_metrics_->spatial_pred_err_h; 80b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_err_v = content_metrics_->spatial_pred_err_v; 81b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 82b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Spatial measure: take average of 3 prediction errors. 83b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_.value = (spatial_err + spatial_err_h + spatial_err_v) / 3.0f; 84b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 85b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Reduce thresholds for large scenes/higher pixel correlation. 86b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float scale2 = image_type_ > kVGA ? kScaleTexture : 1.0; 87b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 88b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (spatial_.value > scale2 * kHighTexture) { 89b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_.level = kHigh; 90b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (spatial_.value < scale2 * kLowTexture) { 91b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_.level = kLow; 92b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else { 93b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_.level = kDefault; 94b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 95b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 96b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 97b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgImageType VCMQmMethod::GetImageType(uint16_t width, 98b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org uint16_t height) { 99b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Get the image type for the encoder frame size. 100b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org uint32_t image_size = width * height; 101b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (image_size == kSizeOfImageType[kQCIF]) { 102b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return kQCIF; 103b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (image_size == kSizeOfImageType[kHCIF]) { 104b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return kHCIF; 105b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (image_size == kSizeOfImageType[kQVGA]) { 106b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return kQVGA; 107b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (image_size == kSizeOfImageType[kCIF]) { 108b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return kCIF; 109b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (image_size == kSizeOfImageType[kHVGA]) { 110b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return kHVGA; 111b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (image_size == kSizeOfImageType[kVGA]) { 112b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return kVGA; 113b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (image_size == kSizeOfImageType[kQFULLHD]) { 114b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return kQFULLHD; 115b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (image_size == kSizeOfImageType[kWHD]) { 116b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return kWHD; 117b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (image_size == kSizeOfImageType[kFULLHD]) { 118b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return kFULLHD; 119b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else { 120b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // No exact match, find closet one. 121b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return FindClosestImageType(width, height); 122b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 123b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 124b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 125b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgImageType VCMQmMethod::FindClosestImageType(uint16_t width, uint16_t height) { 126b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float size = static_cast<float>(width * height); 127b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float min = size; 128b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org int isel = 0; 129b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org for (int i = 0; i < kNumImageTypes; ++i) { 130b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float dist = fabs(size - kSizeOfImageType[i]); 131b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (dist < min) { 132b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org min = dist; 133b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org isel = i; 134b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 135b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 136b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return static_cast<ImageType>(isel); 137b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 138b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 139b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgFrameRateLevelClass VCMQmMethod::FrameRateLevel(float avg_framerate) { 140b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (avg_framerate <= kLowFrameRate) { 141b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return kFrameRateLow; 142b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (avg_framerate <= kMiddleFrameRate) { 143b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return kFrameRateMiddle1; 144b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (avg_framerate <= kHighFrameRate) { 145b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return kFrameRateMiddle2; 146b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else { 147b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return kFrameRateHigh; 148b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 149b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 150b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 151b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// RESOLUTION CLASS 152b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 153b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgVCMQmResolution::VCMQmResolution() 154b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org : qm_(new VCMResolutionScale()) { 155b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org Reset(); 156b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 157b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 158b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgVCMQmResolution::~VCMQmResolution() { 159b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org delete qm_; 160b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 161b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 162b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::ResetRates() { 163b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org sum_target_rate_ = 0.0f; 164b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org sum_incoming_framerate_ = 0.0f; 165b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org sum_rate_MM_ = 0.0f; 166b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org sum_rate_MM_sgn_ = 0.0f; 167b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org sum_packet_loss_ = 0.0f; 168b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org buffer_level_ = kInitBufferLevel * target_bitrate_; 169b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org frame_cnt_ = 0; 170b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org frame_cnt_delta_ = 0; 171b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org low_buffer_cnt_ = 0; 172b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org update_rate_cnt_ = 0; 173b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 174b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 175b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::ResetDownSamplingState() { 176b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org state_dec_factor_spatial_ = 1.0; 177b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org state_dec_factor_temporal_ = 1.0; 178b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org for (int i = 0; i < kDownActionHistorySize; i++) { 179b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[i].spatial = kNoChangeSpatial; 180b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[i].temporal = kNoChangeTemporal; 181b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 182b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 183b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 184b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::Reset() { 185b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org target_bitrate_ = 0.0f; 186b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org incoming_framerate_ = 0.0f; 187b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org buffer_level_ = 0.0f; 188b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org per_frame_bandwidth_ = 0.0f; 189b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_target_rate_ = 0.0f; 190b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_incoming_framerate_ = 0.0f; 191b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_ratio_buffer_low_ = 0.0f; 192b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_rate_mismatch_ = 0.0f; 193b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_rate_mismatch_sgn_ = 0.0f; 194b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_packet_loss_ = 0.0f; 195b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org encoder_state_ = kStableEncoding; 196b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org num_layers_ = 1; 197b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ResetRates(); 198b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ResetDownSamplingState(); 199b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ResetQM(); 200b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 201b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 202b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgEncoderState VCMQmResolution::GetEncoderState() { 203b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return encoder_state_; 204b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 205b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 206b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// Initialize state after re-initializing the encoder, 207b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// i.e., after SetEncodingData() in mediaOpt. 208b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgint VCMQmResolution::Initialize(float bitrate, 209b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float user_framerate, 210b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org uint16_t width, 211b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org uint16_t height, 212b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org int num_layers) { 213b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (user_framerate == 0.0f || width == 0 || height == 0) { 214b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return VCM_PARAMETER_ERROR; 215b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 216b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org Reset(); 217b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org target_bitrate_ = bitrate; 218b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org incoming_framerate_ = user_framerate; 219b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org UpdateCodecParameters(user_framerate, width, height); 220b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org native_width_ = width; 221b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org native_height_ = height; 222b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org native_frame_rate_ = user_framerate; 223b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org num_layers_ = num_layers; 224b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Initial buffer level. 225b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org buffer_level_ = kInitBufferLevel * target_bitrate_; 226b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Per-frame bandwidth. 227b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org per_frame_bandwidth_ = target_bitrate_ / user_framerate; 228b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org init_ = true; 229b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return VCM_OK; 230b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 231b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 232b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::UpdateCodecParameters(float frame_rate, uint16_t width, 233b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org uint16_t height) { 234b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org width_ = width; 235b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org height_ = height; 236b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // |user_frame_rate| is the target frame rate for VPM frame dropper. 237b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org user_frame_rate_ = frame_rate; 238b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org image_type_ = GetImageType(width, height); 239b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 240b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 241b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// Update rate data after every encoded frame. 242b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::UpdateEncodedSize(int encoded_size, 243b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org FrameType encoded_frame_type) { 244b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org frame_cnt_++; 245b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Convert to Kbps. 246b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float encoded_size_kbits = static_cast<float>((encoded_size * 8.0) / 1000.0); 247b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 248b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Update the buffer level: 249b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Note this is not the actual encoder buffer level. 250b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // |buffer_level_| is reset to an initial value after SelectResolution is 251b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // called, and does not account for frame dropping by encoder or VCM. 252b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org buffer_level_ += per_frame_bandwidth_ - encoded_size_kbits; 253b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 254b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Counter for occurrences of low buffer level: 255b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // low/negative values means encoder is likely dropping frames. 256b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (buffer_level_ <= kPercBufferThr * kInitBufferLevel * target_bitrate_) { 257b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org low_buffer_cnt_++; 258b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 259b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 260b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 261b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// Update various quantities after SetTargetRates in MediaOpt. 262b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::UpdateRates(float target_bitrate, 263b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float encoder_sent_rate, 264b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float incoming_framerate, 265b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org uint8_t packet_loss) { 266b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Sum the target bitrate: this is the encoder rate from previous update 267b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // (~1sec), i.e, before the update for next ~1sec. 268b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org sum_target_rate_ += target_bitrate_; 269b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org update_rate_cnt_++; 270b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 271b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Sum the received (from RTCP reports) packet loss rates. 272b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org sum_packet_loss_ += static_cast<float>(packet_loss / 255.0); 273b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 274b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Sum the sequence rate mismatch: 275b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Mismatch here is based on the difference between the target rate 276b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // used (in previous ~1sec) and the average actual encoding rate measured 277b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // at previous ~1sec. 278b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float diff = target_bitrate_ - encoder_sent_rate; 279b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (target_bitrate_ > 0.0) 280b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org sum_rate_MM_ += fabs(diff) / target_bitrate_; 281b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org int sgnDiff = diff > 0 ? 1 : (diff < 0 ? -1 : 0); 282b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // To check for consistent under(+)/over_shooting(-) of target rate. 283b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org sum_rate_MM_sgn_ += sgnDiff; 284b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 285b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Update with the current new target and frame rate: 286b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // these values are ones the encoder will use for the current/next ~1sec. 287b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org target_bitrate_ = target_bitrate; 288b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org incoming_framerate_ = incoming_framerate; 289b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org sum_incoming_framerate_ += incoming_framerate_; 290b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Update the per_frame_bandwidth: 291b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // this is the per_frame_bw for the current/next ~1sec. 292b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org per_frame_bandwidth_ = 0.0f; 293b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (incoming_framerate_ > 0.0f) { 294b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org per_frame_bandwidth_ = target_bitrate_ / incoming_framerate_; 295b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 296b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 297b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 298b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// Select the resolution factors: frame size and frame rate change (qm scales). 299b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// Selection is for going down in resolution, or for going back up 300b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// (if a previous down-sampling action was taken). 301b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 302b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// In the current version the following constraints are imposed: 303b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// 1) We only allow for one action, either down or up, at a given time. 304b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// 2) The possible down-sampling actions are: spatial by 1/2x1/2, 3/4x3/4; 305b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// temporal/frame rate reduction by 1/2 and 2/3. 306b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// 3) The action for going back up is the reverse of last (spatial or temporal) 307b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// down-sampling action. The list of down-sampling actions from the 308b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// Initialize() state are kept in |down_action_history_|. 309b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// 4) The total amount of down-sampling (spatial and/or temporal) from the 310b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// Initialize() state (native resolution) is limited by various factors. 311b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgint VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { 312b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (!init_) { 313b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return VCM_UNINITIALIZED; 314b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 315b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (content_metrics_ == NULL) { 316b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org Reset(); 317b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org *qm = qm_; 318b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return VCM_OK; 319b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 320b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 321b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Check conditions on down-sampling state. 322b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(state_dec_factor_spatial_ >= 1.0f); 323b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(state_dec_factor_temporal_ >= 1.0f); 324b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(state_dec_factor_spatial_ <= kMaxSpatialDown); 325b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(state_dec_factor_temporal_ <= kMaxTempDown); 326b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(state_dec_factor_temporal_ * state_dec_factor_spatial_ <= 327b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kMaxTotalDown); 328b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 329b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Compute content class for selection. 330b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org content_class_ = ComputeContentClass(); 331b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Compute various rate quantities for selection. 332b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ComputeRatesForSelection(); 333b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 334b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Get the encoder state. 335b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ComputeEncoderState(); 336b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 337b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Default settings: no action. 338b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org SetDefaultAction(); 339b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org *qm = qm_; 340b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 341b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Check for going back up in resolution, if we have had some down-sampling 342b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // relative to native state in Initialize(). 343b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (down_action_history_[0].spatial != kNoChangeSpatial || 344b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[0].temporal != kNoChangeTemporal) { 345b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (GoingUpResolution()) { 346b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org *qm = qm_; 347b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return VCM_OK; 348b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 349b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 350b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 351b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Check for going down in resolution. 352b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (GoingDownResolution()) { 353b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org *qm = qm_; 354b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return VCM_OK; 355b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 356b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return VCM_OK; 357b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 358b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 359b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::SetDefaultAction() { 360b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->codec_width = width_; 361b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->codec_height = height_; 362b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->frame_rate = user_frame_rate_; 363b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->change_resolution_spatial = false; 364b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->change_resolution_temporal = false; 365b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_width_fact = 1.0f; 366b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_height_fact = 1.0f; 367b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->temporal_fact = 1.0f; 368b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kNoChangeSpatial; 369b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = kNoChangeTemporal; 370b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 371b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 372b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::ComputeRatesForSelection() { 373b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_target_rate_ = 0.0f; 374b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_incoming_framerate_ = 0.0f; 375b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_ratio_buffer_low_ = 0.0f; 376b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_rate_mismatch_ = 0.0f; 377b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_rate_mismatch_sgn_ = 0.0f; 378b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_packet_loss_ = 0.0f; 379b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (frame_cnt_ > 0) { 380b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_ratio_buffer_low_ = static_cast<float>(low_buffer_cnt_) / 381b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org static_cast<float>(frame_cnt_); 382b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 383b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (update_rate_cnt_ > 0) { 384b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_rate_mismatch_ = static_cast<float>(sum_rate_MM_) / 385b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org static_cast<float>(update_rate_cnt_); 386b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_rate_mismatch_sgn_ = static_cast<float>(sum_rate_MM_sgn_) / 387b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org static_cast<float>(update_rate_cnt_); 388b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_target_rate_ = static_cast<float>(sum_target_rate_) / 389b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org static_cast<float>(update_rate_cnt_); 390b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_incoming_framerate_ = static_cast<float>(sum_incoming_framerate_) / 391b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org static_cast<float>(update_rate_cnt_); 392b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_packet_loss_ = static_cast<float>(sum_packet_loss_) / 393b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org static_cast<float>(update_rate_cnt_); 394b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 395b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // For selection we may want to weight some quantities more heavily 396b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // with the current (i.e., next ~1sec) rate values. 397b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_target_rate_ = kWeightRate * avg_target_rate_ + 398b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org (1.0 - kWeightRate) * target_bitrate_; 399b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_incoming_framerate_ = kWeightRate * avg_incoming_framerate_ + 400b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org (1.0 - kWeightRate) * incoming_framerate_; 401b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Use base layer frame rate for temporal layers: this will favor spatial. 402b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(num_layers_ > 0); 403b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org framerate_level_ = FrameRateLevel( 404b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org avg_incoming_framerate_ / static_cast<float>(1 << (num_layers_ - 1))); 405b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 406b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 407b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::ComputeEncoderState() { 408b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Default. 409b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org encoder_state_ = kStableEncoding; 410b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 411b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Assign stressed state if: 412b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // 1) occurrences of low buffer levels is high, or 413b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // 2) rate mis-match is high, and consistent over-shooting by encoder. 414b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if ((avg_ratio_buffer_low_ > kMaxBufferLow) || 415b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ((avg_rate_mismatch_ > kMaxRateMisMatch) && 416b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org (avg_rate_mismatch_sgn_ < -kRateOverShoot))) { 417b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org encoder_state_ = kStressedEncoding; 418b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 419b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Assign easy state if: 420b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // 1) rate mis-match is high, and 421b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // 2) consistent under-shooting by encoder. 422b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if ((avg_rate_mismatch_ > kMaxRateMisMatch) && 423b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org (avg_rate_mismatch_sgn_ > kRateUnderShoot)) { 424b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org encoder_state_ = kEasyEncoding; 425b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 426b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 427b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 428b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgbool VCMQmResolution::GoingUpResolution() { 429b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // For going up, we check for undoing the previous down-sampling action. 430b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 431b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float fac_width = kFactorWidthSpatial[down_action_history_[0].spatial]; 432b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float fac_height = kFactorHeightSpatial[down_action_history_[0].spatial]; 433b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float fac_temp = kFactorTemporal[down_action_history_[0].temporal]; 434b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // For going up spatially, we allow for going up by 3/4x3/4 at each stage. 435b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // So if the last spatial action was 1/2x1/2 it would be undone in 2 stages. 436b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Modify the fac_width/height for this case. 437b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (down_action_history_[0].spatial == kOneQuarterSpatialUniform) { 438b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org fac_width = kFactorWidthSpatial[kOneQuarterSpatialUniform] / 439b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kFactorWidthSpatial[kOneHalfSpatialUniform]; 440b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org fac_height = kFactorHeightSpatial[kOneQuarterSpatialUniform] / 441b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kFactorHeightSpatial[kOneHalfSpatialUniform]; 442b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 443b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 444b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Check if we should go up both spatially and temporally. 445b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (down_action_history_[0].spatial != kNoChangeSpatial && 446b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[0].temporal != kNoChangeTemporal) { 447b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (ConditionForGoingUp(fac_width, fac_height, fac_temp, 448b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kTransRateScaleUpSpatialTemp)) { 449b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = down_action_history_[0].spatial; 450b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = down_action_history_[0].temporal; 451b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org UpdateDownsamplingState(kUpResolution); 452b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return true; 453b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 454b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 455b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Check if we should go up either spatially or temporally. 456b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org bool selected_up_spatial = false; 457b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org bool selected_up_temporal = false; 458b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (down_action_history_[0].spatial != kNoChangeSpatial) { 459b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org selected_up_spatial = ConditionForGoingUp(fac_width, fac_height, 1.0f, 460b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kTransRateScaleUpSpatial); 461b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 462b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (down_action_history_[0].temporal != kNoChangeTemporal) { 463b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org selected_up_temporal = ConditionForGoingUp(1.0f, 1.0f, fac_temp, 464b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kTransRateScaleUpTemp); 465b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 466b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (selected_up_spatial && !selected_up_temporal) { 467b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = down_action_history_[0].spatial; 468b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = kNoChangeTemporal; 469b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org UpdateDownsamplingState(kUpResolution); 470b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return true; 471b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (!selected_up_spatial && selected_up_temporal) { 472b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kNoChangeSpatial; 473b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = down_action_history_[0].temporal; 474b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org UpdateDownsamplingState(kUpResolution); 475b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return true; 476b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (selected_up_spatial && selected_up_temporal) { 477b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org PickSpatialOrTemporal(); 478b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org UpdateDownsamplingState(kUpResolution); 479b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return true; 480b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 481b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return false; 482b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 483b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 484b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgbool VCMQmResolution::ConditionForGoingUp(float fac_width, 485b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float fac_height, 486b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float fac_temp, 487b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float scale_fac) { 488b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float estimated_transition_rate_up = GetTransitionRate(fac_width, fac_height, 489b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org fac_temp, scale_fac); 490b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Go back up if: 491b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // 1) target rate is above threshold and current encoder state is stable, or 492b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // 2) encoder state is easy (encoder is significantly under-shooting target). 493b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (((avg_target_rate_ > estimated_transition_rate_up) && 494b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org (encoder_state_ == kStableEncoding)) || 495b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org (encoder_state_ == kEasyEncoding)) { 496b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return true; 497b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else { 498b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return false; 499b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 500b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 501b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 502b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgbool VCMQmResolution::GoingDownResolution() { 503b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float estimated_transition_rate_down = 504b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org GetTransitionRate(1.0f, 1.0f, 1.0f, 1.0f); 505b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float max_rate = kFrameRateFac[framerate_level_] * kMaxRateQm[image_type_]; 506b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Resolution reduction if: 507b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // (1) target rate is below transition rate, or 508b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // (2) encoder is in stressed state and target rate below a max threshold. 509b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if ((avg_target_rate_ < estimated_transition_rate_down ) || 510b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org (encoder_state_ == kStressedEncoding && avg_target_rate_ < max_rate)) { 511b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Get the down-sampling action: based on content class, and how low 512b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // average target rate is relative to transition rate. 513b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org uint8_t spatial_fact = 514b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kSpatialAction[content_class_ + 515b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 9 * RateClass(estimated_transition_rate_down)]; 516b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org uint8_t temp_fact = 517b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kTemporalAction[content_class_ + 518b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 9 * RateClass(estimated_transition_rate_down)]; 519b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 520b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org switch (spatial_fact) { 521b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org case 4: { 522b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kOneQuarterSpatialUniform; 523b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org break; 524b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 525b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org case 2: { 526b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kOneHalfSpatialUniform; 527b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org break; 528b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 529b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org case 1: { 530b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kNoChangeSpatial; 531b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org break; 532b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 533b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org default: { 534b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(false); 535b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 536b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 537b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org switch (temp_fact) { 538b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org case 3: { 539b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = kTwoThirdsTemporal; 540b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org break; 541b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 542b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org case 2: { 543b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = kOneHalfTemporal; 544b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org break; 545b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 546b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org case 1: { 547b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = kNoChangeTemporal; 548b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org break; 549b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 550b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org default: { 551b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(false); 552b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 553b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 554b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Only allow for one action (spatial or temporal) at a given time. 555b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(action_.temporal == kNoChangeTemporal || 556b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial == kNoChangeSpatial); 557b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 558b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Adjust cases not captured in tables, mainly based on frame rate, and 559b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // also check for odd frame sizes. 560b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org AdjustAction(); 561b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 562b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Update down-sampling state. 563b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.spatial != kNoChangeSpatial || 564b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal != kNoChangeTemporal) { 565b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org UpdateDownsamplingState(kDownResolution); 566b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return true; 567b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 568b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 569b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return false; 570b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 571b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 572b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgfloat VCMQmResolution::GetTransitionRate(float fac_width, 573b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float fac_height, 574b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float fac_temp, 575b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float scale_fac) { 576b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ImageType image_type = GetImageType( 577b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org static_cast<uint16_t>(fac_width * width_), 578b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org static_cast<uint16_t>(fac_height * height_)); 579b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 580b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org FrameRateLevelClass framerate_level = 581b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org FrameRateLevel(fac_temp * avg_incoming_framerate_); 582b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // If we are checking for going up temporally, and this is the last 583b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // temporal action, then use native frame rate. 584b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (down_action_history_[1].temporal == kNoChangeTemporal && 585b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org fac_temp > 1.0f) { 586b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org framerate_level = FrameRateLevel(native_frame_rate_); 587b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 588b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 589b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // The maximum allowed rate below which down-sampling is allowed: 590b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Nominal values based on image format (frame size and frame rate). 591b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float max_rate = kFrameRateFac[framerate_level] * kMaxRateQm[image_type]; 592b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 593b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org uint8_t image_class = image_type > kVGA ? 1: 0; 594b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org uint8_t table_index = image_class * 9 + content_class_; 595b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Scale factor for down-sampling transition threshold: 596b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // factor based on the content class and the image size. 597b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float scaleTransRate = kScaleTransRateQm[table_index]; 598b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Threshold bitrate for resolution action. 599b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return static_cast<float> (scale_fac * scaleTransRate * max_rate); 600b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 601b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 602b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::UpdateDownsamplingState(UpDownAction up_down) { 603b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (up_down == kUpResolution) { 604b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_width_fact = 1.0f / kFactorWidthSpatial[action_.spatial]; 605b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_height_fact = 1.0f / kFactorHeightSpatial[action_.spatial]; 606b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // If last spatial action was 1/2x1/2, we undo it in two steps, so the 607b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // spatial scale factor in this first step is modified as (4.0/3.0 / 2.0). 608b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.spatial == kOneQuarterSpatialUniform) { 609b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_width_fact = 610b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 1.0f * kFactorWidthSpatial[kOneHalfSpatialUniform] / 611b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kFactorWidthSpatial[kOneQuarterSpatialUniform]; 612b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_height_fact = 613b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 1.0f * kFactorHeightSpatial[kOneHalfSpatialUniform] / 614b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kFactorHeightSpatial[kOneQuarterSpatialUniform]; 615b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 616b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->temporal_fact = 1.0f / kFactorTemporal[action_.temporal]; 617b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org RemoveLastDownAction(); 618b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (up_down == kDownResolution) { 619b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ConstrainAmountOfDownSampling(); 620b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ConvertSpatialFractionalToWhole(); 621b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_width_fact = kFactorWidthSpatial[action_.spatial]; 622b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_height_fact = kFactorHeightSpatial[action_.spatial]; 623b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->temporal_fact = kFactorTemporal[action_.temporal]; 624b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org InsertLatestDownAction(); 625b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else { 626b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // This function should only be called if either the Up or Down action 627b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // has been selected. 628b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(false); 629b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 630b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org UpdateCodecResolution(); 631b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org state_dec_factor_spatial_ = state_dec_factor_spatial_ * 632b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_width_fact * qm_->spatial_height_fact; 633b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org state_dec_factor_temporal_ = state_dec_factor_temporal_ * qm_->temporal_fact; 634b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 635b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 636b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::UpdateCodecResolution() { 637b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.spatial != kNoChangeSpatial) { 638b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->change_resolution_spatial = true; 639b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->codec_width = static_cast<uint16_t>(width_ / 640b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_width_fact + 0.5f); 641b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->codec_height = static_cast<uint16_t>(height_ / 642b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_height_fact + 0.5f); 643b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Size should not exceed native sizes. 644b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(qm_->codec_width <= native_width_); 645b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(qm_->codec_height <= native_height_); 646b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // New sizes should be multiple of 2, otherwise spatial should not have 647b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // been selected. 648b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(qm_->codec_width % 2 == 0); 649b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(qm_->codec_height % 2 == 0); 650b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 651b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.temporal != kNoChangeTemporal) { 652b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->change_resolution_temporal = true; 653b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Update the frame rate based on the average incoming frame rate. 654b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->frame_rate = avg_incoming_framerate_ / qm_->temporal_fact + 0.5f; 655b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (down_action_history_[0].temporal == 0) { 656b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // When we undo the last temporal-down action, make sure we go back up 657b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // to the native frame rate. Since the incoming frame rate may 658b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // fluctuate over time, |avg_incoming_framerate_| scaled back up may 659b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // be smaller than |native_frame rate_|. 660b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->frame_rate = native_frame_rate_; 661b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 662b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 663b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 664b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 665b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orguint8_t VCMQmResolution::RateClass(float transition_rate) { 666b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return avg_target_rate_ < (kFacLowRate * transition_rate) ? 0: 667b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org (avg_target_rate_ >= transition_rate ? 2 : 1); 668b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 669b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 670b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// TODO(marpan): Would be better to capture these frame rate adjustments by 671b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// extending the table data (qm_select_data.h). 672b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::AdjustAction() { 673b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // If the spatial level is default state (neither low or high), motion level 674b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // is not high, and spatial action was selected, switch to 2/3 frame rate 675b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // reduction if the average incoming frame rate is high. 676b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (spatial_.level == kDefault && motion_.level != kHigh && 677b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial != kNoChangeSpatial && 678b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org framerate_level_ == kFrameRateHigh) { 679b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kNoChangeSpatial; 680b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = kTwoThirdsTemporal; 681b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 682b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // If both motion and spatial level are low, and temporal down action was 683b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // selected, switch to spatial 3/4x3/4 if the frame rate is not above the 684b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // lower middle level (|kFrameRateMiddle1|). 685b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (motion_.level == kLow && spatial_.level == kLow && 686b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org framerate_level_ <= kFrameRateMiddle1 && 687b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal != kNoChangeTemporal) { 688b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kOneHalfSpatialUniform; 689b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = kNoChangeTemporal; 690b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 691b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // If spatial action is selected, and there has been too much spatial 692b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // reduction already (i.e., 1/4), then switch to temporal action if the 693b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // average frame rate is not low. 694b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.spatial != kNoChangeSpatial && 695b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[0].spatial == kOneQuarterSpatialUniform && 696b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org framerate_level_ != kFrameRateLow) { 697b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kNoChangeSpatial; 698b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = kTwoThirdsTemporal; 699b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 700b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Never use temporal action if number of temporal layers is above 2. 701b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (num_layers_ > 2) { 702b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.temporal != kNoChangeTemporal) { 703b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kOneHalfSpatialUniform; 704b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 705b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = kNoChangeTemporal; 706b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 707b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // If spatial action was selected, we need to make sure the frame sizes 708b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // are multiples of two. Otherwise switch to 2/3 temporal. 709b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.spatial != kNoChangeSpatial && 710b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org !EvenFrameSize()) { 711b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kNoChangeSpatial; 712b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Only one action (spatial or temporal) is allowed at a given time, so need 713b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // to check whether temporal action is currently selected. 714b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = kTwoThirdsTemporal; 715b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 716b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 717b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 718b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::ConvertSpatialFractionalToWhole() { 719b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // If 3/4 spatial is selected, check if there has been another 3/4, 720b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // and if so, combine them into 1/2. 1/2 scaling is more efficient than 9/16. 721b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Note we define 3/4x3/4 spatial as kOneHalfSpatialUniform. 722b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.spatial == kOneHalfSpatialUniform) { 723b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org bool found = false; 724b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org int isel = kDownActionHistorySize; 725b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org for (int i = 0; i < kDownActionHistorySize; ++i) { 726b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (down_action_history_[i].spatial == kOneHalfSpatialUniform) { 727b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org isel = i; 728b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org found = true; 729b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org break; 730b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 731b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 732b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (found) { 733b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kOneQuarterSpatialUniform; 734b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org state_dec_factor_spatial_ = state_dec_factor_spatial_ / 735b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org (kFactorWidthSpatial[kOneHalfSpatialUniform] * 736b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kFactorHeightSpatial[kOneHalfSpatialUniform]); 737b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Check if switching to 1/2x1/2 (=1/4) spatial is allowed. 738b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ConstrainAmountOfDownSampling(); 739b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.spatial == kNoChangeSpatial) { 740b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Not allowed. Go back to 3/4x3/4 spatial. 741b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kOneHalfSpatialUniform; 742b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org state_dec_factor_spatial_ = state_dec_factor_spatial_ * 743b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kFactorWidthSpatial[kOneHalfSpatialUniform] * 744b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kFactorHeightSpatial[kOneHalfSpatialUniform]; 745b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else { 746b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Switching is allowed. Remove 3/4x3/4 from the history, and update 747b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // the frame size. 748b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org for (int i = isel; i < kDownActionHistorySize - 1; ++i) { 749b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[i].spatial = 750b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[i + 1].spatial; 751b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 752b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org width_ = width_ * kFactorWidthSpatial[kOneHalfSpatialUniform]; 753b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org height_ = height_ * kFactorHeightSpatial[kOneHalfSpatialUniform]; 754b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 755b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 756b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 757b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 758b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 759b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// Returns false if the new frame sizes, under the current spatial action, 760b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// are not multiples of two. 761b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgbool VCMQmResolution::EvenFrameSize() { 762b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.spatial == kOneHalfSpatialUniform) { 763b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if ((width_ * 3 / 4) % 2 != 0 || (height_ * 3 / 4) % 2 != 0) { 764b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return false; 765b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 766b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (action_.spatial == kOneQuarterSpatialUniform) { 767b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if ((width_ * 1 / 2) % 2 != 0 || (height_ * 1 / 2) % 2 != 0) { 768b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return false; 769b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 770b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 771b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return true; 772b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 773b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 774b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::InsertLatestDownAction() { 775b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.spatial != kNoChangeSpatial) { 776b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org for (int i = kDownActionHistorySize - 1; i > 0; --i) { 777b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[i].spatial = down_action_history_[i - 1].spatial; 778b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 779b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[0].spatial = action_.spatial; 780b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 781b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.temporal != kNoChangeTemporal) { 782b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org for (int i = kDownActionHistorySize - 1; i > 0; --i) { 783b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[i].temporal = down_action_history_[i - 1].temporal; 784b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 785b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[0].temporal = action_.temporal; 786b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 787b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 788b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 789b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::RemoveLastDownAction() { 790b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.spatial != kNoChangeSpatial) { 791b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // If the last spatial action was 1/2x1/2 we replace it with 3/4x3/4. 792b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.spatial == kOneQuarterSpatialUniform) { 793b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[0].spatial = kOneHalfSpatialUniform; 794b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else { 795b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org for (int i = 0; i < kDownActionHistorySize - 1; ++i) { 796b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[i].spatial = down_action_history_[i + 1].spatial; 797b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 798b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[kDownActionHistorySize - 1].spatial = 799b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kNoChangeSpatial; 800b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 801b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 802b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.temporal != kNoChangeTemporal) { 803b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org for (int i = 0; i < kDownActionHistorySize - 1; ++i) { 804b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[i].temporal = down_action_history_[i + 1].temporal; 805b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 806b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org down_action_history_[kDownActionHistorySize - 1].temporal = 807b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org kNoChangeTemporal; 808b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 809b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 810b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 811b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::ConstrainAmountOfDownSampling() { 812b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Sanity checks on down-sampling selection: 813b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // override the settings for too small image size and/or frame rate. 814b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Also check the limit on current down-sampling states. 815b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 816b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float spatial_width_fact = kFactorWidthSpatial[action_.spatial]; 817b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float spatial_height_fact = kFactorHeightSpatial[action_.spatial]; 818b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float temporal_fact = kFactorTemporal[action_.temporal]; 819b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float new_dec_factor_spatial = state_dec_factor_spatial_ * 820b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_width_fact * spatial_height_fact; 821b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float new_dec_factor_temp = state_dec_factor_temporal_ * temporal_fact; 822b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 823b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // No spatial sampling if current frame size is too small, or if the 824b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // amount of spatial down-sampling is above maximum spatial down-action. 825b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if ((width_ * height_) <= kMinImageSize || 826b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org new_dec_factor_spatial > kMaxSpatialDown) { 827b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kNoChangeSpatial; 828b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org new_dec_factor_spatial = state_dec_factor_spatial_; 829b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 830b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // No frame rate reduction if average frame rate is below some point, or if 831b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // the amount of temporal down-sampling is above maximum temporal down-action. 832b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (avg_incoming_framerate_ <= kMinFrameRate || 833b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org new_dec_factor_temp > kMaxTempDown) { 834b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = kNoChangeTemporal; 835b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org new_dec_factor_temp = state_dec_factor_temporal_; 836b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 837b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Check if the total (spatial-temporal) down-action is above maximum allowed, 838b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // if so, disallow the current selected down-action. 839b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (new_dec_factor_spatial * new_dec_factor_temp > kMaxTotalDown) { 840b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (action_.spatial != kNoChangeSpatial) { 841b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kNoChangeSpatial; 842b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else if (action_.temporal != kNoChangeTemporal) { 843b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = kNoChangeTemporal; 844b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else { 845b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // We only allow for one action (spatial or temporal) at a given time, so 846b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // either spatial or temporal action is selected when this function is 847b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // called. If the selected action is disallowed from one of the above 848b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // 2 prior conditions (on spatial & temporal max down-action), then this 849b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // condition "total down-action > |kMaxTotalDown|" would not be entered. 850b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org assert(false); 851b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 852b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 853b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 854b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 855b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::PickSpatialOrTemporal() { 856b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Pick the one that has had the most down-sampling thus far. 857b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (state_dec_factor_spatial_ > state_dec_factor_temporal_) { 858b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = down_action_history_[0].spatial; 859b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = kNoChangeTemporal; 860b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } else { 861b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.spatial = kNoChangeSpatial; 862b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org action_.temporal = down_action_history_[0].temporal; 863b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 864b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 865b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 866b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// TODO(marpan): Update when we allow for directional spatial down-sampling. 867b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmResolution::SelectSpatialDirectionMode(float transition_rate) { 868b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Default is 4/3x4/3 869b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // For bit rates well below transitional rate, we select 2x2. 870b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (avg_target_rate_ < transition_rate * kRateRedSpatial2X2) { 871b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_width_fact = 2.0f; 872b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_height_fact = 2.0f; 873b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 874b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Otherwise check prediction errors and aspect ratio. 875b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float spatial_err = 0.0f; 876b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float spatial_err_h = 0.0f; 877b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float spatial_err_v = 0.0f; 878b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (content_metrics_) { 879b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_err = content_metrics_->spatial_pred_err; 880b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_err_h = content_metrics_->spatial_pred_err_h; 881b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_err_v = content_metrics_->spatial_pred_err_v; 882b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 883b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 884b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Favor 1x2 if aspect_ratio is 16:9. 885b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (aspect_ratio_ >= 16.0f / 9.0f) { 886b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Check if 1x2 has lowest prediction error. 887b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (spatial_err_h < spatial_err && spatial_err_h < spatial_err_v) { 888b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_width_fact = 2.0f; 889b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_height_fact = 1.0f; 890b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 891b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 892b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Check for 4/3x4/3 selection: favor 2x2 over 1x2 and 2x1. 893b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (spatial_err < spatial_err_h * (1.0f + kSpatialErr2x2VsHoriz) && 894b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_err < spatial_err_v * (1.0f + kSpatialErr2X2VsVert)) { 895b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_width_fact = 4.0f / 3.0f; 896b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_height_fact = 4.0f / 3.0f; 897b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 898b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Check for 2x1 selection. 899b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (spatial_err_v < spatial_err_h * (1.0f - kSpatialErrVertVsHoriz) && 900b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org spatial_err_v < spatial_err * (1.0f - kSpatialErr2X2VsVert)) { 901b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_width_fact = 1.0f; 902b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org qm_->spatial_height_fact = 2.0f; 903b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 904b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 905b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 906b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// ROBUSTNESS CLASS 907b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 908b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgVCMQmRobustness::VCMQmRobustness() { 909b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org Reset(); 910b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 911b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 912b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgVCMQmRobustness::~VCMQmRobustness() { 913b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 914b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 915b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid VCMQmRobustness::Reset() { 916b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org prev_total_rate_ = 0.0f; 917b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org prev_rtt_time_ = 0; 918b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org prev_packet_loss_ = 0; 919b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org prev_code_rate_delta_ = 0; 920b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ResetQM(); 921b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 922b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 923b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// Adjust the FEC rate based on the content and the network state 924b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// (packet loss rate, total rate/bandwidth, round trip time). 925b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// Note that packetLoss here is the filtered loss value. 926b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgfloat VCMQmRobustness::AdjustFecFactor(uint8_t code_rate_delta, 927b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float total_rate, 928b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float framerate, 929b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org uint32_t rtt_time, 930b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org uint8_t packet_loss) { 931b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Default: no adjustment 932b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float adjust_fec = 1.0f; 933b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (content_metrics_ == NULL) { 934b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return adjust_fec; 935b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 936b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Compute class state of the content. 937b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ComputeMotionNFD(); 938b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ComputeSpatial(); 939b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 940b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // TODO(marpan): Set FEC adjustment factor. 941b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 942b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Keep track of previous values of network state: 943b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // adjustment may be also based on pattern of changes in network state. 944b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org prev_total_rate_ = total_rate; 945b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org prev_rtt_time_ = rtt_time; 946b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org prev_packet_loss_ = packet_loss; 947b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org prev_code_rate_delta_ = code_rate_delta; 948b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return adjust_fec; 949b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 950b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 951b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org// Set the UEP (unequal-protection across packets) on/off for the FEC. 952b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgbool VCMQmRobustness::SetUepProtection(uint8_t code_rate_delta, 953b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float total_rate, 954b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org uint8_t packet_loss, 955b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org bool frame_type) { 956b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Default. 957b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return false; 958b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 9593b89e10f31160da35b408fd00cb8f89d2b08862dpbos@webrtc.org} // namespace 960