1470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com/* 29d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com * 4470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com * Use of this source code is governed by a BSD-style license 5470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com * that can be found in the LICENSE file in the root of the source 6470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com * tree. An additional intellectual property rights grant can be found 7470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com * in the file PATENTS. All contributing project authors may 8470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com * be found in the AUTHORS file in the root of the source tree. 9470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com */ 10470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 112557b86e7648ffebc5781df9f093ca5a84efc219Henrik Kjellander#include "webrtc/modules/video_coding/qm_select.h" 12470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 13470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com#include <math.h> 14470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 15ff761fba8274d93bd73e76c8b8a1f2d0776dd840Henrik Kjellander#include "webrtc/modules/include/module_common_types.h" 162557b86e7648ffebc5781df9f093ca5a84efc219Henrik Kjellander#include "webrtc/modules/video_coding/include/video_coding_defines.h" 172557b86e7648ffebc5781df9f093ca5a84efc219Henrik Kjellander#include "webrtc/modules/video_coding/internal_defines.h" 182557b86e7648ffebc5781df9f093ca5a84efc219Henrik Kjellander#include "webrtc/modules/video_coding/qm_select_data.h" 1998f53510b222f71fdd8b799b2f33737ceeb28c61Henrik Kjellander#include "webrtc/system_wrappers/include/trace.h" 209d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 21470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.comnamespace webrtc { 22470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 2386548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com// QM-METHOD class 2486548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com 2586548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.comVCMQmMethod::VCMQmMethod() 26accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org : content_metrics_(NULL), 27accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org width_(0), 28accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org height_(0), 29e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org user_frame_rate_(0.0f), 30accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org native_width_(0), 31accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org native_height_(0), 32e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org native_frame_rate_(0.0f), 33cb8050c16613ead32c6147ebfaae7b8d1a6d0351marpan@webrtc.org image_type_(kVGA), 34e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org framerate_level_(kFrameRateHigh), 35accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org init_(false) { 366584e5800131dbda94acac8dd4533b356afaf1bamarpan@webrtc.org ResetQM(); 37470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com} 38470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 399d3ab61325c5ed216ea52bc829f1d8c81347459bphilipelVCMQmMethod::~VCMQmMethod() {} 40470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 419d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgvoid VCMQmMethod::ResetQM() { 42accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org aspect_ratio_ = 1.0f; 43accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org motion_.Reset(); 44accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org spatial_.Reset(); 45accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org content_class_ = 0; 46470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com} 47470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 489d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orguint8_t VCMQmMethod::ComputeContentClass() { 499d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org ComputeMotionNFD(); 509d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org ComputeSpatial(); 51accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return content_class_ = 3 * motion_.level + spatial_.level; 5286548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com} 5386548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com 549d3ab61325c5ed216ea52bc829f1d8c81347459bphilipelvoid VCMQmMethod::UpdateContent(const VideoContentMetrics* contentMetrics) { 55accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org content_metrics_ = contentMetrics; 5686548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com} 5786548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com 589d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgvoid VCMQmMethod::ComputeMotionNFD() { 59accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (content_metrics_) { 60accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org motion_.value = content_metrics_->motion_magnitude; 619d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 629d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Determine motion level. 63accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (motion_.value < kLowMotionNfd) { 64accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org motion_.level = kLow; 65accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else if (motion_.value > kHighMotionNfd) { 669d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel motion_.level = kHigh; 679d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } else { 68accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org motion_.level = kDefault; 699d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 70bd5648f2db2c143584cc2c55df99d11453e13818marpan@webrtc.org} 71bd5648f2db2c143584cc2c55df99d11453e13818marpan@webrtc.org 729d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgvoid VCMQmMethod::ComputeSpatial() { 73accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float spatial_err = 0.0; 74accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float spatial_err_h = 0.0; 75accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float spatial_err_v = 0.0; 76accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (content_metrics_) { 779d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel spatial_err = content_metrics_->spatial_pred_err; 78accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org spatial_err_h = content_metrics_->spatial_pred_err_h; 79accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org spatial_err_v = content_metrics_->spatial_pred_err_v; 809d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 819d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Spatial measure: take average of 3 prediction errors. 82accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org spatial_.value = (spatial_err + spatial_err_h + spatial_err_v) / 3.0f; 839d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 84accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Reduce thresholds for large scenes/higher pixel correlation. 85accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float scale2 = image_type_ > kVGA ? kScaleTexture : 1.0; 869d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 87accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (spatial_.value > scale2 * kHighTexture) { 88accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org spatial_.level = kHigh; 89accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else if (spatial_.value < scale2 * kLowTexture) { 90accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org spatial_.level = kLow; 919d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } else { 92accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org spatial_.level = kDefault; 93accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 94accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org} 95accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 969d3ab61325c5ed216ea52bc829f1d8c81347459bphilipelImageType VCMQmMethod::GetImageType(uint16_t width, uint16_t height) { 97accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Get the image type for the encoder frame size. 98accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint32_t image_size = width * height; 99accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (image_size == kSizeOfImageType[kQCIF]) { 100accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return kQCIF; 101accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else if (image_size == kSizeOfImageType[kHCIF]) { 102accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return kHCIF; 103accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else if (image_size == kSizeOfImageType[kQVGA]) { 104accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return kQVGA; 105accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else if (image_size == kSizeOfImageType[kCIF]) { 106accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return kCIF; 107accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else if (image_size == kSizeOfImageType[kHVGA]) { 108accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return kHVGA; 109accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else if (image_size == kSizeOfImageType[kVGA]) { 110accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return kVGA; 111accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else if (image_size == kSizeOfImageType[kQFULLHD]) { 112accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return kQFULLHD; 113accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else if (image_size == kSizeOfImageType[kWHD]) { 114accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return kWHD; 115accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else if (image_size == kSizeOfImageType[kFULLHD]) { 116accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return kFULLHD; 1179d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } else { 118accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // No exact match, find closet one. 119accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return FindClosestImageType(width, height); 1209d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 121bd5648f2db2c143584cc2c55df99d11453e13818marpan@webrtc.org} 122bd5648f2db2c143584cc2c55df99d11453e13818marpan@webrtc.org 123accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgImageType VCMQmMethod::FindClosestImageType(uint16_t width, uint16_t height) { 124accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float size = static_cast<float>(width * height); 125accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float min = size; 126accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org int isel = 0; 127accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org for (int i = 0; i < kNumImageTypes; ++i) { 128accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float dist = fabs(size - kSizeOfImageType[i]); 129accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (dist < min) { 130accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org min = dist; 131accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org isel = i; 132accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 133accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 134accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return static_cast<ImageType>(isel); 135accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org} 136accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 137e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.orgFrameRateLevelClass VCMQmMethod::FrameRateLevel(float avg_framerate) { 138c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org if (avg_framerate <= kLowFrameRate) { 139e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org return kFrameRateLow; 140c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org } else if (avg_framerate <= kMiddleFrameRate) { 141e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org return kFrameRateMiddle1; 142c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org } else if (avg_framerate <= kHighFrameRate) { 1439d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel return kFrameRateMiddle2; 1449d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } else { 145e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org return kFrameRateHigh; 1469d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 14786548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com} 14886548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com 1499d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org// RESOLUTION CLASS 15086548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com 1519d3ab61325c5ed216ea52bc829f1d8c81347459bphilipelVCMQmResolution::VCMQmResolution() : qm_(new VCMResolutionScale()) { 1529d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org Reset(); 153470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com} 154470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 1559d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgVCMQmResolution::~VCMQmResolution() { 156accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org delete qm_; 157470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com} 158470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 1599d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgvoid VCMQmResolution::ResetRates() { 160accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org sum_target_rate_ = 0.0f; 161accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org sum_incoming_framerate_ = 0.0f; 162accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org sum_rate_MM_ = 0.0f; 163accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org sum_rate_MM_sgn_ = 0.0f; 164accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org sum_packet_loss_ = 0.0f; 165c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org buffer_level_ = kInitBufferLevel * target_bitrate_; 166accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org frame_cnt_ = 0; 167accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org frame_cnt_delta_ = 0; 168accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org low_buffer_cnt_ = 0; 169accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org update_rate_cnt_ = 0; 1709d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 171470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 1729d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgvoid VCMQmResolution::ResetDownSamplingState() { 173accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org state_dec_factor_spatial_ = 1.0; 1749d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel state_dec_factor_temporal_ = 1.0; 175accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org for (int i = 0; i < kDownActionHistorySize; i++) { 176accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org down_action_history_[i].spatial = kNoChangeSpatial; 177accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org down_action_history_[i].temporal = kNoChangeTemporal; 178accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 1799d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 180470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 1819d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgvoid VCMQmResolution::Reset() { 182accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org target_bitrate_ = 0.0f; 183accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org incoming_framerate_ = 0.0f; 184accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org buffer_level_ = 0.0f; 185c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org per_frame_bandwidth_ = 0.0f; 186accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_target_rate_ = 0.0f; 187accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_incoming_framerate_ = 0.0f; 188accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_ratio_buffer_low_ = 0.0f; 189accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_rate_mismatch_ = 0.0f; 190accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_rate_mismatch_sgn_ = 0.0f; 191accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_packet_loss_ = 0.0f; 192accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org encoder_state_ = kStableEncoding; 193accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org num_layers_ = 1; 1949d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org ResetRates(); 1959d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org ResetDownSamplingState(); 1969d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org ResetQM(); 1979d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 198470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 1999d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgEncoderState VCMQmResolution::GetEncoderState() { 200accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return encoder_state_; 2019d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 202470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 2039d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org// Initialize state after re-initializing the encoder, 2049d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org// i.e., after SetEncodingData() in mediaOpt. 205accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgint VCMQmResolution::Initialize(float bitrate, 206accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float user_framerate, 2079d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org uint16_t width, 208accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint16_t height, 209accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org int num_layers) { 210accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (user_framerate == 0.0f || width == 0 || height == 0) { 2119d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org return VCM_PARAMETER_ERROR; 2129d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 2139d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org Reset(); 214accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org target_bitrate_ = bitrate; 215accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org incoming_framerate_ = user_framerate; 216e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org UpdateCodecParameters(user_framerate, width, height); 217accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org native_width_ = width; 218accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org native_height_ = height; 219e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org native_frame_rate_ = user_framerate; 220accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org num_layers_ = num_layers; 2219d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Initial buffer level. 222c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org buffer_level_ = kInitBufferLevel * target_bitrate_; 2239d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Per-frame bandwidth. 224e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org per_frame_bandwidth_ = target_bitrate_ / user_framerate; 2259d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel init_ = true; 2269d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org return VCM_OK; 2279d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 228bd5648f2db2c143584cc2c55df99d11453e13818marpan@webrtc.org 2299d3ab61325c5ed216ea52bc829f1d8c81347459bphilipelvoid VCMQmResolution::UpdateCodecParameters(float frame_rate, 2309d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel uint16_t width, 231e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org uint16_t height) { 232accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org width_ = width; 233accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org height_ = height; 234e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // |user_frame_rate| is the target frame rate for VPM frame dropper. 235e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org user_frame_rate_ = frame_rate; 236accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org image_type_ = GetImageType(width, height); 237470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com} 238470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 2399d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org// Update rate data after every encoded frame. 240273a414b0ec2e58fdf3b817ad8b1a02f4ce15287pbos@webrtc.orgvoid VCMQmResolution::UpdateEncodedSize(size_t encoded_size) { 241accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org frame_cnt_++; 2429d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Convert to Kbps. 2434591fbd09f9cb6e83433c49a12dd8524c2806502pkasting@chromium.org float encoded_size_kbits = 8.0f * static_cast<float>(encoded_size) / 1000.0f; 2449d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2459d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Update the buffer level: 2469d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Note this is not the actual encoder buffer level. 247c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // |buffer_level_| is reset to an initial value after SelectResolution is 248accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // called, and does not account for frame dropping by encoder or VCM. 249accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org buffer_level_ += per_frame_bandwidth_ - encoded_size_kbits; 250c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org 2519d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Counter for occurrences of low buffer level: 2529d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // low/negative values means encoder is likely dropping frames. 253c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org if (buffer_level_ <= kPercBufferThr * kInitBufferLevel * target_bitrate_) { 254accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org low_buffer_cnt_++; 2559d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 2569d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 257470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 2589d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org// Update various quantities after SetTargetRates in MediaOpt. 259accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgvoid VCMQmResolution::UpdateRates(float target_bitrate, 260accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float encoder_sent_rate, 261accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float incoming_framerate, 262accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint8_t packet_loss) { 263e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // Sum the target bitrate: this is the encoder rate from previous update 264e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // (~1sec), i.e, before the update for next ~1sec. 265accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org sum_target_rate_ += target_bitrate_; 266accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org update_rate_cnt_++; 2679d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2689d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Sum the received (from RTCP reports) packet loss rates. 269accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org sum_packet_loss_ += static_cast<float>(packet_loss / 255.0); 2709d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2719d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Sum the sequence rate mismatch: 2729d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Mismatch here is based on the difference between the target rate 2739d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // used (in previous ~1sec) and the average actual encoding rate measured 2749d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // at previous ~1sec. 275accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float diff = target_bitrate_ - encoder_sent_rate; 276accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (target_bitrate_ > 0.0) 277accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org sum_rate_MM_ += fabs(diff) / target_bitrate_; 2789d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org int sgnDiff = diff > 0 ? 1 : (diff < 0 ? -1 : 0); 2799d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // To check for consistent under(+)/over_shooting(-) of target rate. 280accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org sum_rate_MM_sgn_ += sgnDiff; 2819d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2829d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Update with the current new target and frame rate: 283c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // these values are ones the encoder will use for the current/next ~1sec. 2849d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel target_bitrate_ = target_bitrate; 285accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org incoming_framerate_ = incoming_framerate; 286e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org sum_incoming_framerate_ += incoming_framerate_; 2879d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Update the per_frame_bandwidth: 288c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // this is the per_frame_bw for the current/next ~1sec. 2899d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel per_frame_bandwidth_ = 0.0f; 290accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (incoming_framerate_ > 0.0f) { 291accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org per_frame_bandwidth_ = target_bitrate_ / incoming_framerate_; 2929d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 2939d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 294470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 2959d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org// Select the resolution factors: frame size and frame rate change (qm scales). 2969d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org// Selection is for going down in resolution, or for going back up 2979d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org// (if a previous down-sampling action was taken). 2989d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2999d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org// In the current version the following constraints are imposed: 300accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org// 1) We only allow for one action, either down or up, at a given time. 301e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org// 2) The possible down-sampling actions are: spatial by 1/2x1/2, 3/4x3/4; 302e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org// temporal/frame rate reduction by 1/2 and 2/3. 303accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org// 3) The action for going back up is the reverse of last (spatial or temporal) 304accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org// down-sampling action. The list of down-sampling actions from the 305accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org// Initialize() state are kept in |down_action_history_|. 306accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org// 4) The total amount of down-sampling (spatial and/or temporal) from the 307accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org// Initialize() state (native resolution) is limited by various factors. 3089d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgint VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { 309accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (!init_) { 3109d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org return VCM_UNINITIALIZED; 3119d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 312accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (content_metrics_ == NULL) { 3139d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org Reset(); 3149d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel *qm = qm_; 3159d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org return VCM_OK; 3169d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 317470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 318c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // Check conditions on down-sampling state. 319c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org assert(state_dec_factor_spatial_ >= 1.0f); 320c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org assert(state_dec_factor_temporal_ >= 1.0f); 321c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org assert(state_dec_factor_spatial_ <= kMaxSpatialDown); 322c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org assert(state_dec_factor_temporal_ <= kMaxTempDown); 323c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org assert(state_dec_factor_temporal_ * state_dec_factor_spatial_ <= 324c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org kMaxTotalDown); 325c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org 3269d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Compute content class for selection. 327accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org content_class_ = ComputeContentClass(); 3289d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Compute various rate quantities for selection. 3299d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org ComputeRatesForSelection(); 330470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 3319d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Get the encoder state. 3329d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org ComputeEncoderState(); 333470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 334e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // Default settings: no action. 335e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org SetDefaultAction(); 336e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org *qm = qm_; 337e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org 3389d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Check for going back up in resolution, if we have had some down-sampling 339accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // relative to native state in Initialize(). 340accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (down_action_history_[0].spatial != kNoChangeSpatial || 341accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org down_action_history_[0].temporal != kNoChangeTemporal) { 3429d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org if (GoingUpResolution()) { 343accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org *qm = qm_; 3449d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org return VCM_OK; 345470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com } 3469d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 347bd5648f2db2c143584cc2c55df99d11453e13818marpan@webrtc.org 348c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // Check for going down in resolution. 349c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org if (GoingDownResolution()) { 350c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org *qm = qm_; 351c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org return VCM_OK; 3529d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 3539d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org return VCM_OK; 3549d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 355bd5648f2db2c143584cc2c55df99d11453e13818marpan@webrtc.org 356accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgvoid VCMQmResolution::SetDefaultAction() { 357e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org qm_->codec_width = width_; 358e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org qm_->codec_height = height_; 359e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org qm_->frame_rate = user_frame_rate_; 360e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org qm_->change_resolution_spatial = false; 361e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org qm_->change_resolution_temporal = false; 362accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_width_fact = 1.0f; 363accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_height_fact = 1.0f; 364accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->temporal_fact = 1.0f; 365accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.spatial = kNoChangeSpatial; 366accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.temporal = kNoChangeTemporal; 367accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org} 368accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 3699d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgvoid VCMQmResolution::ComputeRatesForSelection() { 370accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_target_rate_ = 0.0f; 371accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_incoming_framerate_ = 0.0f; 372accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_ratio_buffer_low_ = 0.0f; 373accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_rate_mismatch_ = 0.0f; 374accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_rate_mismatch_sgn_ = 0.0f; 375accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_packet_loss_ = 0.0f; 376accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (frame_cnt_ > 0) { 3779d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel avg_ratio_buffer_low_ = 3789d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel static_cast<float>(low_buffer_cnt_) / static_cast<float>(frame_cnt_); 379accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 380accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (update_rate_cnt_ > 0) { 3819d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel avg_rate_mismatch_ = 3829d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel static_cast<float>(sum_rate_MM_) / static_cast<float>(update_rate_cnt_); 383accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_rate_mismatch_sgn_ = static_cast<float>(sum_rate_MM_sgn_) / 3849d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel static_cast<float>(update_rate_cnt_); 385accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_target_rate_ = static_cast<float>(sum_target_rate_) / 3869d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel static_cast<float>(update_rate_cnt_); 387accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_incoming_framerate_ = static_cast<float>(sum_incoming_framerate_) / 3889d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel static_cast<float>(update_rate_cnt_); 3899d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel avg_packet_loss_ = static_cast<float>(sum_packet_loss_) / 3909d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel static_cast<float>(update_rate_cnt_); 3919d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 3929d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // For selection we may want to weight some quantities more heavily 3939d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // with the current (i.e., next ~1sec) rate values. 3949d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel avg_target_rate_ = 3959d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kWeightRate * avg_target_rate_ + (1.0 - kWeightRate) * target_bitrate_; 396accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org avg_incoming_framerate_ = kWeightRate * avg_incoming_framerate_ + 3979d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel (1.0 - kWeightRate) * incoming_framerate_; 398e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // Use base layer frame rate for temporal layers: this will favor spatial. 3993fe3252cb3ffa7f14f7905d668ccf6e6fd277f90marpan@webrtc.org assert(num_layers_ > 0); 4009d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel framerate_level_ = FrameRateLevel(avg_incoming_framerate_ / 4019d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel static_cast<float>(1 << (num_layers_ - 1))); 4029d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 403bd5648f2db2c143584cc2c55df99d11453e13818marpan@webrtc.org 4049d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgvoid VCMQmResolution::ComputeEncoderState() { 4059d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Default. 406accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org encoder_state_ = kStableEncoding; 4079d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 4089d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Assign stressed state if: 4099d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // 1) occurrences of low buffer levels is high, or 4109d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // 2) rate mis-match is high, and consistent over-shooting by encoder. 411accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if ((avg_ratio_buffer_low_ > kMaxBufferLow) || 412accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org ((avg_rate_mismatch_ > kMaxRateMisMatch) && 4139d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel (avg_rate_mismatch_sgn_ < -kRateOverShoot))) { 414accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org encoder_state_ = kStressedEncoding; 4159d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 4169d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Assign easy state if: 4179d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // 1) rate mis-match is high, and 4189d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // 2) consistent under-shooting by encoder. 419accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if ((avg_rate_mismatch_ > kMaxRateMisMatch) && 420accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org (avg_rate_mismatch_sgn_ > kRateUnderShoot)) { 421accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org encoder_state_ = kEasyEncoding; 4229d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 423470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com} 424470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 4259d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgbool VCMQmResolution::GoingUpResolution() { 426accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // For going up, we check for undoing the previous down-sampling action. 427c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org 428accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float fac_width = kFactorWidthSpatial[down_action_history_[0].spatial]; 429accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float fac_height = kFactorHeightSpatial[down_action_history_[0].spatial]; 430accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float fac_temp = kFactorTemporal[down_action_history_[0].temporal]; 431c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // For going up spatially, we allow for going up by 3/4x3/4 at each stage. 432c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // So if the last spatial action was 1/2x1/2 it would be undone in 2 stages. 433c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // Modify the fac_width/height for this case. 434c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org if (down_action_history_[0].spatial == kOneQuarterSpatialUniform) { 435c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org fac_width = kFactorWidthSpatial[kOneQuarterSpatialUniform] / 4369d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kFactorWidthSpatial[kOneHalfSpatialUniform]; 437c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org fac_height = kFactorHeightSpatial[kOneQuarterSpatialUniform] / 4389d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kFactorHeightSpatial[kOneHalfSpatialUniform]; 439c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org } 440accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 4419d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Check if we should go up both spatially and temporally. 442accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (down_action_history_[0].spatial != kNoChangeSpatial && 443accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org down_action_history_[0].temporal != kNoChangeTemporal) { 444accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (ConditionForGoingUp(fac_width, fac_height, fac_temp, 445accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org kTransRateScaleUpSpatialTemp)) { 446accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.spatial = down_action_history_[0].spatial; 447accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.temporal = down_action_history_[0].temporal; 4489d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org UpdateDownsamplingState(kUpResolution); 4499d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org return true; 4509d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 4519d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 452accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Check if we should go up either spatially or temporally. 453accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org bool selected_up_spatial = false; 454accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org bool selected_up_temporal = false; 455accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (down_action_history_[0].spatial != kNoChangeSpatial) { 456accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org selected_up_spatial = ConditionForGoingUp(fac_width, fac_height, 1.0f, 457accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org kTransRateScaleUpSpatial); 458accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 459accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (down_action_history_[0].temporal != kNoChangeTemporal) { 4609d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel selected_up_temporal = 4619d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel ConditionForGoingUp(1.0f, 1.0f, fac_temp, kTransRateScaleUpTemp); 462accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 463accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (selected_up_spatial && !selected_up_temporal) { 464accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.spatial = down_action_history_[0].spatial; 465accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.temporal = kNoChangeTemporal; 466accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org UpdateDownsamplingState(kUpResolution); 467accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return true; 468accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else if (!selected_up_spatial && selected_up_temporal) { 469accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.spatial = kNoChangeSpatial; 470accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.temporal = down_action_history_[0].temporal; 471accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org UpdateDownsamplingState(kUpResolution); 472accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return true; 473accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else if (selected_up_spatial && selected_up_temporal) { 474accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org PickSpatialOrTemporal(); 475accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org UpdateDownsamplingState(kUpResolution); 476accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return true; 477accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 4789d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org return false; 4799d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 480470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 481accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgbool VCMQmResolution::ConditionForGoingUp(float fac_width, 482accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float fac_height, 483accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float fac_temp, 484accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float scale_fac) { 4859d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel float estimated_transition_rate_up = 4869d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel GetTransitionRate(fac_width, fac_height, fac_temp, scale_fac); 4879d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Go back up if: 4889d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // 1) target rate is above threshold and current encoder state is stable, or 4899d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // 2) encoder state is easy (encoder is significantly under-shooting target). 490accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (((avg_target_rate_ > estimated_transition_rate_up) && 4919d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel (encoder_state_ == kStableEncoding)) || 492accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org (encoder_state_ == kEasyEncoding)) { 4939d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org return true; 4949d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } else { 4959d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org return false; 4969d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 4979d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 498470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 4999d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgbool VCMQmResolution::GoingDownResolution() { 500accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float estimated_transition_rate_down = 501accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org GetTransitionRate(1.0f, 1.0f, 1.0f, 1.0f); 502accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float max_rate = kFrameRateFac[framerate_level_] * kMaxRateQm[image_type_]; 5039d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Resolution reduction if: 5049d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // (1) target rate is below transition rate, or 5059d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // (2) encoder is in stressed state and target rate below a max threshold. 5069d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel if ((avg_target_rate_ < estimated_transition_rate_down) || 507accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org (encoder_state_ == kStressedEncoding && avg_target_rate_ < max_rate)) { 508accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Get the down-sampling action: based on content class, and how low 509accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // average target rate is relative to transition rate. 510accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint8_t spatial_fact = 511accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org kSpatialAction[content_class_ + 512accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 9 * RateClass(estimated_transition_rate_down)]; 513accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint8_t temp_fact = 514accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org kTemporalAction[content_class_ + 515accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 9 * RateClass(estimated_transition_rate_down)]; 516accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 517accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org switch (spatial_fact) { 5189d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org case 4: { 519accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.spatial = kOneQuarterSpatialUniform; 5209d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org break; 5219d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 5229d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org case 2: { 523accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.spatial = kOneHalfSpatialUniform; 5249d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org break; 5259d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 5269d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org case 1: { 527accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.spatial = kNoChangeSpatial; 5289d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org break; 5299d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 5309d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel default: { assert(false); } 5319d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 532accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org switch (temp_fact) { 533accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org case 3: { 534accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.temporal = kTwoThirdsTemporal; 535accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org break; 536accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 5379d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org case 2: { 538accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.temporal = kOneHalfTemporal; 5399d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org break; 5409d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 5419d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org case 1: { 542accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.temporal = kNoChangeTemporal; 5439d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org break; 5449d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 5459d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel default: { assert(false); } 5469d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 547e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // Only allow for one action (spatial or temporal) at a given time. 548e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org assert(action_.temporal == kNoChangeTemporal || 549e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org action_.spatial == kNoChangeSpatial); 5509d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 551c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // Adjust cases not captured in tables, mainly based on frame rate, and 552c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // also check for odd frame sizes. 553accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org AdjustAction(); 554accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 5559d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Update down-sampling state. 556accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (action_.spatial != kNoChangeSpatial || 557accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.temporal != kNoChangeTemporal) { 5589d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org UpdateDownsamplingState(kDownResolution); 5599d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org return true; 5609d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 5619d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 5629d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org return false; 5639d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 564bd5648f2db2c143584cc2c55df99d11453e13818marpan@webrtc.org 565accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgfloat VCMQmResolution::GetTransitionRate(float fac_width, 566accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float fac_height, 567accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float fac_temp, 568accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float scale_fac) { 5699d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel ImageType image_type = 5709d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel GetImageType(static_cast<uint16_t>(fac_width * width_), 5719d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel static_cast<uint16_t>(fac_height * height_)); 572accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 573e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org FrameRateLevelClass framerate_level = 574accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org FrameRateLevel(fac_temp * avg_incoming_framerate_); 575e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // If we are checking for going up temporally, and this is the last 576e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // temporal action, then use native frame rate. 577e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org if (down_action_history_[1].temporal == kNoChangeTemporal && 578e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org fac_temp > 1.0f) { 579e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org framerate_level = FrameRateLevel(native_frame_rate_); 580e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org } 5819d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 5829d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // The maximum allowed rate below which down-sampling is allowed: 5839d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Nominal values based on image format (frame size and frame rate). 584accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float max_rate = kFrameRateFac[framerate_level] * kMaxRateQm[image_type]; 5859d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 5869d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel uint8_t image_class = image_type > kVGA ? 1 : 0; 587accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint8_t table_index = image_class * 9 + content_class_; 5889d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Scale factor for down-sampling transition threshold: 5899d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // factor based on the content class and the image size. 590accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float scaleTransRate = kScaleTransRateQm[table_index]; 5919d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Threshold bitrate for resolution action. 5929d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel return static_cast<float>(scale_fac * scaleTransRate * max_rate); 593accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org} 594accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 595accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgvoid VCMQmResolution::UpdateDownsamplingState(UpDownAction up_down) { 596accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (up_down == kUpResolution) { 597accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_width_fact = 1.0f / kFactorWidthSpatial[action_.spatial]; 598accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_height_fact = 1.0f / kFactorHeightSpatial[action_.spatial]; 599c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // If last spatial action was 1/2x1/2, we undo it in two steps, so the 600c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // spatial scale factor in this first step is modified as (4.0/3.0 / 2.0). 601c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org if (action_.spatial == kOneQuarterSpatialUniform) { 6029d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel qm_->spatial_width_fact = 1.0f * 6039d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kFactorWidthSpatial[kOneHalfSpatialUniform] / 6049d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kFactorWidthSpatial[kOneQuarterSpatialUniform]; 605c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org qm_->spatial_height_fact = 606c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org 1.0f * kFactorHeightSpatial[kOneHalfSpatialUniform] / 607c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org kFactorHeightSpatial[kOneQuarterSpatialUniform]; 608c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org } 609accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->temporal_fact = 1.0f / kFactorTemporal[action_.temporal]; 610accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org RemoveLastDownAction(); 611accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else if (up_down == kDownResolution) { 612c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org ConstrainAmountOfDownSampling(); 613c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org ConvertSpatialFractionalToWhole(); 614accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_width_fact = kFactorWidthSpatial[action_.spatial]; 615accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_height_fact = kFactorHeightSpatial[action_.spatial]; 616accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->temporal_fact = kFactorTemporal[action_.temporal]; 617accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org InsertLatestDownAction(); 618accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else { 619accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // This function should only be called if either the Up or Down action 620accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // has been selected. 621accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org assert(false); 622accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 623e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org UpdateCodecResolution(); 624accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org state_dec_factor_spatial_ = state_dec_factor_spatial_ * 6259d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel qm_->spatial_width_fact * 6269d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel qm_->spatial_height_fact; 627accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org state_dec_factor_temporal_ = state_dec_factor_temporal_ * qm_->temporal_fact; 628accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org} 629accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 6309d3ab61325c5ed216ea52bc829f1d8c81347459bphilipelvoid VCMQmResolution::UpdateCodecResolution() { 631e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org if (action_.spatial != kNoChangeSpatial) { 632e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org qm_->change_resolution_spatial = true; 6339d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel qm_->codec_width = 6349d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel static_cast<uint16_t>(width_ / qm_->spatial_width_fact + 0.5f); 6359d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel qm_->codec_height = 6369d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel static_cast<uint16_t>(height_ / qm_->spatial_height_fact + 0.5f); 637c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // Size should not exceed native sizes. 638e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org assert(qm_->codec_width <= native_width_); 639e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org assert(qm_->codec_height <= native_height_); 640c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // New sizes should be multiple of 2, otherwise spatial should not have 641c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // been selected. 642e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org assert(qm_->codec_width % 2 == 0); 643e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org assert(qm_->codec_height % 2 == 0); 644e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org } 645e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org if (action_.temporal != kNoChangeTemporal) { 646e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org qm_->change_resolution_temporal = true; 647e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // Update the frame rate based on the average incoming frame rate. 648e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org qm_->frame_rate = avg_incoming_framerate_ / qm_->temporal_fact + 0.5f; 649e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org if (down_action_history_[0].temporal == 0) { 650e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // When we undo the last temporal-down action, make sure we go back up 651e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // to the native frame rate. Since the incoming frame rate may 652e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // fluctuate over time, |avg_incoming_framerate_| scaled back up may 653e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // be smaller than |native_frame rate_|. 654e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org qm_->frame_rate = native_frame_rate_; 655e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org } 656e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org } 657e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org} 658e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org 659accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orguint8_t VCMQmResolution::RateClass(float transition_rate) { 6609d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel return avg_target_rate_ < (kFacLowRate * transition_rate) 6619d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel ? 0 6629d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel : (avg_target_rate_ >= transition_rate ? 2 : 1); 663accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org} 664accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 665c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org// TODO(marpan): Would be better to capture these frame rate adjustments by 666c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org// extending the table data (qm_select_data.h). 667accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgvoid VCMQmResolution::AdjustAction() { 668c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // If the spatial level is default state (neither low or high), motion level 669c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // is not high, and spatial action was selected, switch to 2/3 frame rate 670c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // reduction if the average incoming frame rate is high. 671accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (spatial_.level == kDefault && motion_.level != kHigh && 672c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org action_.spatial != kNoChangeSpatial && 673e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org framerate_level_ == kFrameRateHigh) { 674accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.spatial = kNoChangeSpatial; 675c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org action_.temporal = kTwoThirdsTemporal; 676accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 677c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // If both motion and spatial level are low, and temporal down action was 678c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // selected, switch to spatial 3/4x3/4 if the frame rate is not above the 679c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // lower middle level (|kFrameRateMiddle1|). 680accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (motion_.level == kLow && spatial_.level == kLow && 681c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org framerate_level_ <= kFrameRateMiddle1 && 682e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org action_.temporal != kNoChangeTemporal) { 683accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.spatial = kOneHalfSpatialUniform; 684accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.temporal = kNoChangeTemporal; 685accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 686c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // If spatial action is selected, and there has been too much spatial 687c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // reduction already (i.e., 1/4), then switch to temporal action if the 688c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // average frame rate is not low. 689c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org if (action_.spatial != kNoChangeSpatial && 690e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org down_action_history_[0].spatial == kOneQuarterSpatialUniform && 691e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org framerate_level_ != kFrameRateLow) { 692e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org action_.spatial = kNoChangeSpatial; 693c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org action_.temporal = kTwoThirdsTemporal; 694e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org } 695e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // Never use temporal action if number of temporal layers is above 2. 696e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org if (num_layers_ > 2) { 6979d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel if (action_.temporal != kNoChangeTemporal) { 698e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org action_.spatial = kOneHalfSpatialUniform; 699e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org } 700e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org action_.temporal = kNoChangeTemporal; 701e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org } 702c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // If spatial action was selected, we need to make sure the frame sizes 703c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // are multiples of two. Otherwise switch to 2/3 temporal. 7049d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel if (action_.spatial != kNoChangeSpatial && !EvenFrameSize()) { 705c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org action_.spatial = kNoChangeSpatial; 706c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // Only one action (spatial or temporal) is allowed at a given time, so need 707c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // to check whether temporal action is currently selected. 708c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org action_.temporal = kTwoThirdsTemporal; 709c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org } 710e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org} 711e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org 712e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.orgvoid VCMQmResolution::ConvertSpatialFractionalToWhole() { 713e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // If 3/4 spatial is selected, check if there has been another 3/4, 714e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // and if so, combine them into 1/2. 1/2 scaling is more efficient than 9/16. 715e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // Note we define 3/4x3/4 spatial as kOneHalfSpatialUniform. 716e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org if (action_.spatial == kOneHalfSpatialUniform) { 717e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org bool found = false; 718e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org int isel = kDownActionHistorySize; 719e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org for (int i = 0; i < kDownActionHistorySize; ++i) { 7209d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel if (down_action_history_[i].spatial == kOneHalfSpatialUniform) { 721e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org isel = i; 722e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org found = true; 723e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org break; 724e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org } 725e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org } 726e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org if (found) { 7279d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel action_.spatial = kOneQuarterSpatialUniform; 7289d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel state_dec_factor_spatial_ = 7299d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel state_dec_factor_spatial_ / 7309d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel (kFactorWidthSpatial[kOneHalfSpatialUniform] * 7319d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kFactorHeightSpatial[kOneHalfSpatialUniform]); 7329d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel // Check if switching to 1/2x1/2 (=1/4) spatial is allowed. 7339d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel ConstrainAmountOfDownSampling(); 7349d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel if (action_.spatial == kNoChangeSpatial) { 7359d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel // Not allowed. Go back to 3/4x3/4 spatial. 7369d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel action_.spatial = kOneHalfSpatialUniform; 7379d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel state_dec_factor_spatial_ = 7389d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel state_dec_factor_spatial_ * 7399d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kFactorWidthSpatial[kOneHalfSpatialUniform] * 7409d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kFactorHeightSpatial[kOneHalfSpatialUniform]; 7419d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel } else { 7429d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel // Switching is allowed. Remove 3/4x3/4 from the history, and update 7439d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel // the frame size. 7449d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel for (int i = isel; i < kDownActionHistorySize - 1; ++i) { 7459d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel down_action_history_[i].spatial = down_action_history_[i + 1].spatial; 7469d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel } 7479d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel width_ = width_ * kFactorWidthSpatial[kOneHalfSpatialUniform]; 7489d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel height_ = height_ * kFactorHeightSpatial[kOneHalfSpatialUniform]; 7499d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel } 750e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org } 751e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org } 752accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org} 753accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 754c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org// Returns false if the new frame sizes, under the current spatial action, 755c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org// are not multiples of two. 756c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.orgbool VCMQmResolution::EvenFrameSize() { 757accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (action_.spatial == kOneHalfSpatialUniform) { 758c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org if ((width_ * 3 / 4) % 2 != 0 || (height_ * 3 / 4) % 2 != 0) { 759c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org return false; 760c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org } 761c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org } else if (action_.spatial == kOneQuarterSpatialUniform) { 762c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org if ((width_ * 1 / 2) % 2 != 0 || (height_ * 1 / 2) % 2 != 0) { 763c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org return false; 764accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 765accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 766c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org return true; 767470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com} 768470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 769accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgvoid VCMQmResolution::InsertLatestDownAction() { 770accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (action_.spatial != kNoChangeSpatial) { 771accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org for (int i = kDownActionHistorySize - 1; i > 0; --i) { 772c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org down_action_history_[i].spatial = down_action_history_[i - 1].spatial; 7739d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 774accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org down_action_history_[0].spatial = action_.spatial; 775accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 776accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (action_.temporal != kNoChangeTemporal) { 777accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org for (int i = kDownActionHistorySize - 1; i > 0; --i) { 778c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org down_action_history_[i].temporal = down_action_history_[i - 1].temporal; 7799d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 780accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org down_action_history_[0].temporal = action_.temporal; 7819d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 7829d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 783470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 784accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgvoid VCMQmResolution::RemoveLastDownAction() { 785accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (action_.spatial != kNoChangeSpatial) { 786c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // If the last spatial action was 1/2x1/2 we replace it with 3/4x3/4. 787c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org if (action_.spatial == kOneQuarterSpatialUniform) { 788c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org down_action_history_[0].spatial = kOneHalfSpatialUniform; 789c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org } else { 790c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org for (int i = 0; i < kDownActionHistorySize - 1; ++i) { 791c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org down_action_history_[i].spatial = down_action_history_[i + 1].spatial; 792c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org } 793c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org down_action_history_[kDownActionHistorySize - 1].spatial = 794c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org kNoChangeSpatial; 795accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 796accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 797accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (action_.temporal != kNoChangeTemporal) { 798c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org for (int i = 0; i < kDownActionHistorySize - 1; ++i) { 799c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org down_action_history_[i].temporal = down_action_history_[i + 1].temporal; 800accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 801accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org down_action_history_[kDownActionHistorySize - 1].temporal = 802accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org kNoChangeTemporal; 803accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 804accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org} 805accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 806accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgvoid VCMQmResolution::ConstrainAmountOfDownSampling() { 807accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Sanity checks on down-sampling selection: 808accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // override the settings for too small image size and/or frame rate. 809accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Also check the limit on current down-sampling states. 810accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 811c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org float spatial_width_fact = kFactorWidthSpatial[action_.spatial]; 812c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org float spatial_height_fact = kFactorHeightSpatial[action_.spatial]; 813c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org float temporal_fact = kFactorTemporal[action_.temporal]; 8149d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel float new_dec_factor_spatial = 8159d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel state_dec_factor_spatial_ * spatial_width_fact * spatial_height_fact; 816c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org float new_dec_factor_temp = state_dec_factor_temporal_ * temporal_fact; 817c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org 818c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // No spatial sampling if current frame size is too small, or if the 819c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // amount of spatial down-sampling is above maximum spatial down-action. 820accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if ((width_ * height_) <= kMinImageSize || 821accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org new_dec_factor_spatial > kMaxSpatialDown) { 822accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.spatial = kNoChangeSpatial; 823c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org new_dec_factor_spatial = state_dec_factor_spatial_; 824accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 825c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // No frame rate reduction if average frame rate is below some point, or if 826c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // the amount of temporal down-sampling is above maximum temporal down-action. 827accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (avg_incoming_framerate_ <= kMinFrameRate || 828c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org new_dec_factor_temp > kMaxTempDown) { 829accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.temporal = kNoChangeTemporal; 830c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org new_dec_factor_temp = state_dec_factor_temporal_; 831c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org } 832c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // Check if the total (spatial-temporal) down-action is above maximum allowed, 833c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // if so, disallow the current selected down-action. 834c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org if (new_dec_factor_spatial * new_dec_factor_temp > kMaxTotalDown) { 835c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org if (action_.spatial != kNoChangeSpatial) { 836c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org action_.spatial = kNoChangeSpatial; 837c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org } else if (action_.temporal != kNoChangeTemporal) { 838c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org action_.temporal = kNoChangeTemporal; 839c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org } else { 840c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // We only allow for one action (spatial or temporal) at a given time, so 841c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // either spatial or temporal action is selected when this function is 842c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // called. If the selected action is disallowed from one of the above 843c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // 2 prior conditions (on spatial & temporal max down-action), then this 844c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // condition "total down-action > |kMaxTotalDown|" would not be entered. 845c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org assert(false); 846c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org } 847accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } 848accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org} 849accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 850accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgvoid VCMQmResolution::PickSpatialOrTemporal() { 851accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Pick the one that has had the most down-sampling thus far. 852accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (state_dec_factor_spatial_ > state_dec_factor_temporal_) { 853accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.spatial = down_action_history_[0].spatial; 854accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.temporal = kNoChangeTemporal; 855accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org } else { 856accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.spatial = kNoChangeSpatial; 857accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org action_.temporal = down_action_history_[0].temporal; 8589d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 8599d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 8609d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 861e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org// TODO(marpan): Update when we allow for directional spatial down-sampling. 862accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgvoid VCMQmResolution::SelectSpatialDirectionMode(float transition_rate) { 863accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Default is 4/3x4/3 8649d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // For bit rates well below transitional rate, we select 2x2. 865accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (avg_target_rate_ < transition_rate * kRateRedSpatial2X2) { 866accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_width_fact = 2.0f; 867accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_height_fact = 2.0f; 8689d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 8699d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Otherwise check prediction errors and aspect ratio. 870accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float spatial_err = 0.0f; 871accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float spatial_err_h = 0.0f; 872accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float spatial_err_v = 0.0f; 873accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (content_metrics_) { 874accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org spatial_err = content_metrics_->spatial_pred_err; 875accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org spatial_err_h = content_metrics_->spatial_pred_err_h; 876accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org spatial_err_v = content_metrics_->spatial_pred_err_v; 8779d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 8789d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 8799d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Favor 1x2 if aspect_ratio is 16:9. 880accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (aspect_ratio_ >= 16.0f / 9.0f) { 8819d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Check if 1x2 has lowest prediction error. 882accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (spatial_err_h < spatial_err && spatial_err_h < spatial_err_v) { 883accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_width_fact = 2.0f; 884accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_height_fact = 1.0f; 8859d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 8869d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 887accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Check for 4/3x4/3 selection: favor 2x2 over 1x2 and 2x1. 888accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (spatial_err < spatial_err_h * (1.0f + kSpatialErr2x2VsHoriz) && 889accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org spatial_err < spatial_err_v * (1.0f + kSpatialErr2X2VsVert)) { 890accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_width_fact = 4.0f / 3.0f; 891accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_height_fact = 4.0f / 3.0f; 8929d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 8939d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Check for 2x1 selection. 894accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (spatial_err_v < spatial_err_h * (1.0f - kSpatialErrVertVsHoriz) && 895accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org spatial_err_v < spatial_err * (1.0f - kSpatialErr2X2VsVert)) { 896accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_width_fact = 1.0f; 897accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org qm_->spatial_height_fact = 2.0f; 8989d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 8999d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org} 900470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 90186548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com// ROBUSTNESS CLASS 902470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 9039d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgVCMQmRobustness::VCMQmRobustness() { 9049d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org Reset(); 905470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com} 906470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 9079d3ab61325c5ed216ea52bc829f1d8c81347459bphilipelVCMQmRobustness::~VCMQmRobustness() {} 908470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 9099d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgvoid VCMQmRobustness::Reset() { 910accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org prev_total_rate_ = 0.0f; 911accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org prev_rtt_time_ = 0; 912accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org prev_packet_loss_ = 0; 913accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org prev_code_rate_delta_ = 0; 9149d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org ResetQM(); 91586548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com} 916470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 91786548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com// Adjust the FEC rate based on the content and the network state 91886548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com// (packet loss rate, total rate/bandwidth, round trip time). 91986548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com// Note that packetLoss here is the filtered loss value. 920accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgfloat VCMQmRobustness::AdjustFecFactor(uint8_t code_rate_delta, 921accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float total_rate, 922accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float framerate, 92316825b1a828bb4ff40f7682040e43a239b7b8ca3pkasting@chromium.org int64_t rtt_time, 924accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint8_t packet_loss) { 9259d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Default: no adjustment 9269d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel float adjust_fec = 1.0f; 927accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org if (content_metrics_ == NULL) { 928accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return adjust_fec; 9299d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 9309d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Compute class state of the content. 9319d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org ComputeMotionNFD(); 9329d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org ComputeSpatial(); 9339d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 9349d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // TODO(marpan): Set FEC adjustment factor. 9359d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 9369d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Keep track of previous values of network state: 9379d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // adjustment may be also based on pattern of changes in network state. 938accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org prev_total_rate_ = total_rate; 939accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org prev_rtt_time_ = rtt_time; 940accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org prev_packet_loss_ = packet_loss; 941accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org prev_code_rate_delta_ = code_rate_delta; 942accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org return adjust_fec; 94386548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com} 94486548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com 9459d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org// Set the UEP (unequal-protection across packets) on/off for the FEC. 946accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgbool VCMQmRobustness::SetUepProtection(uint8_t code_rate_delta, 947accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float total_rate, 948accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint8_t packet_loss, 949accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org bool frame_type) { 9509d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Default. 9519d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org return false; 952470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com} 9539d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel} // namespace webrtc 954