1470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com/* 29d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com * 4470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com * Use of this source code is governed by a BSD-style license 5470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com * that can be found in the LICENSE file in the root of the source 6470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com * tree. An additional intellectual property rights grant can be found 7470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com * in the file PATENTS. All contributing project authors may 8470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com * be found in the AUTHORS file in the root of the source tree. 9470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com */ 10470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 11470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com#ifndef WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_ 12470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com#define WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_ 13470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 14a4407329d4e16eab3d2e87cde0824188c06acb5apbos@webrtc.org#include "webrtc/common_types.h" 15a4407329d4e16eab3d2e87cde0824188c06acb5apbos@webrtc.org#include "webrtc/typedefs.h" 169d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1786548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com/******************************************************/ 1886548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com/* Quality Modes: Resolution and Robustness settings */ 1986548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com/******************************************************/ 20470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 219d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgnamespace webrtc { 22470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.comstruct VideoContentMetrics; 23470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 249d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgstruct VCMResolutionScale { 259d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org VCMResolutionScale() 26e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org : codec_width(640), 27e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org codec_height(480), 28e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org frame_rate(30.0f), 29e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org spatial_width_fact(1.0f), 30accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org spatial_height_fact(1.0f), 31accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org temporal_fact(1.0f), 32e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org change_resolution_spatial(false), 339d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel change_resolution_temporal(false) {} 34e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org uint16_t codec_width; 35e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org uint16_t codec_height; 36e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org float frame_rate; 37accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float spatial_width_fact; 38accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float spatial_height_fact; 39accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float temporal_fact; 40e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org bool change_resolution_spatial; 41e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org bool change_resolution_temporal; 42470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com}; 43470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 44accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgenum ImageType { 459d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kQCIF = 0, // 176x144 469d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kHCIF, // 264x216 = half(~3/4x3/4) CIF. 479d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kQVGA, // 320x240 = quarter VGA. 489d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kCIF, // 352x288 499d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kHVGA, // 480x360 = half(~3/4x3/4) VGA. 509d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kVGA, // 640x480 519d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kQFULLHD, // 960x540 = quarter FULLHD, and half(~3/4x3/4) WHD. 529d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kWHD, // 1280x720 539d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kFULLHD, // 1920x1080 54accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org kNumImageTypes 55accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org}; 56accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 579d3ab61325c5ed216ea52bc829f1d8c81347459bphilipelconst uint32_t kSizeOfImageType[kNumImageTypes] = { 589d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel 25344, 57024, 76800, 101376, 172800, 307200, 518400, 921600, 2073600}; 59accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 60e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.orgenum FrameRateLevelClass { 61e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org kFrameRateLow, 62e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org kFrameRateMiddle1, 63e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org kFrameRateMiddle2, 64e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org kFrameRateHigh 65e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org}; 66e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org 679d3ab61325c5ed216ea52bc829f1d8c81347459bphilipelenum ContentLevelClass { kLow, kHigh, kDefault }; 68470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 699d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgstruct VCMContFeature { 709d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel VCMContFeature() : value(0.0f), level(kDefault) {} 719d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org void Reset() { 729d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org value = 0.0f; 739d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org level = kDefault; 749d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org } 759d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org float value; 76e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org ContentLevelClass level; 779d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org}; 78470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 799d3ab61325c5ed216ea52bc829f1d8c81347459bphilipelenum UpDownAction { kUpResolution, kDownResolution }; 80470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 81accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgenum SpatialAction { 82accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org kNoChangeSpatial, 839d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kOneHalfSpatialUniform, // 3/4 x 3/4: 9/6 ~1/2 pixel reduction. 849d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kOneQuarterSpatialUniform, // 1/2 x 1/2: 1/4 pixel reduction. 85accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org kNumModesSpatial 86accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org}; 87accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 88accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgenum TemporalAction { 89accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org kNoChangeTemporal, 909d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kTwoThirdsTemporal, // 2/3 frame rate reduction 919d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel kOneHalfTemporal, // 1/2 frame rate reduction 92accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org kNumModesTemporal 93accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org}; 94accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 95accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.orgstruct ResolutionAction { 969d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel ResolutionAction() : spatial(kNoChangeSpatial), temporal(kNoChangeTemporal) {} 97accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org SpatialAction spatial; 98accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org TemporalAction temporal; 99accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org}; 100accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 101accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org// Down-sampling factors for spatial (width and height), and temporal. 1029d3ab61325c5ed216ea52bc829f1d8c81347459bphilipelconst float kFactorWidthSpatial[kNumModesSpatial] = {1.0f, 4.0f / 3.0f, 2.0f}; 103accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 1049d3ab61325c5ed216ea52bc829f1d8c81347459bphilipelconst float kFactorHeightSpatial[kNumModesSpatial] = {1.0f, 4.0f / 3.0f, 2.0f}; 105accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 1069d3ab61325c5ed216ea52bc829f1d8c81347459bphilipelconst float kFactorTemporal[kNumModesTemporal] = {1.0f, 1.5f, 2.0f}; 107accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 1089d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgenum EncoderState { 1099d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org kStableEncoding, // Low rate mis-match, stable buffer levels. 1109d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org kStressedEncoding, // Significant over-shooting of target rate, 1119d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Buffer under-flow, etc. 1129d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org kEasyEncoding // Significant under-shooting of target rate. 113470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com}; 114470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 11586548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com// QmMethod class: main class for resolution and robustness settings 11686548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com 1179d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgclass VCMQmMethod { 1189d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org public: 1199d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org VCMQmMethod(); 1209d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org virtual ~VCMQmMethod(); 1219d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1229d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Reset values 1239d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org void ResetQM(); 1249d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org virtual void Reset() = 0; 1259d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1269d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Compute content class. 1279d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org uint8_t ComputeContentClass(); 1289d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1299d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Update with the content metrics. 130accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org void UpdateContent(const VideoContentMetrics* content_metrics); 1319d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1329d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Compute spatial texture magnitude and level. 1339d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Spatial texture is a spatial prediction error measure. 1349d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org void ComputeSpatial(); 1359d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1369d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Compute motion magnitude and level for NFD metric. 1379d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // NFD is normalized frame difference (normalized by spatial variance). 1389d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org void ComputeMotionNFD(); 1399d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1409d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Get the imageType (CIF, VGA, HD, etc) for the system width/height. 141accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org ImageType GetImageType(uint16_t width, uint16_t height); 142accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 143accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Return the closest image type. 144accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org ImageType FindClosestImageType(uint16_t width, uint16_t height); 1459d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1469d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Get the frame rate level. 147e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org FrameRateLevelClass FrameRateLevel(float frame_rate); 1489d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1499d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org protected: 1509d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Content Data. 151accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org const VideoContentMetrics* content_metrics_; 1529d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1539d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Encoder frame sizes and native frame sizes. 154accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint16_t width_; 155accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint16_t height_; 156e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org float user_frame_rate_; 157accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint16_t native_width_; 158accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint16_t native_height_; 159e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org float native_frame_rate_; 160accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float aspect_ratio_; 1619d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Image type and frame rate leve, for the current encoder resolution. 162accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org ImageType image_type_; 163e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org FrameRateLevelClass framerate_level_; 1649d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Content class data. 165accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org VCMContFeature motion_; 166accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org VCMContFeature spatial_; 167accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint8_t content_class_; 168accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org bool init_; 16986548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com}; 17086548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com 17186548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com// Resolution settings class 17286548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com 1739d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgclass VCMQmResolution : public VCMQmMethod { 1749d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org public: 1759d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org VCMQmResolution(); 1769d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org virtual ~VCMQmResolution(); 1779d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1789d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Reset all quantities. 1799d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org virtual void Reset(); 1809d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1819d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Reset rate quantities and counters after every SelectResolution() call. 1829d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org void ResetRates(); 1839d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1849d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Reset down-sampling state. 1859d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org void ResetDownSamplingState(); 1869d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1879d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Get the encoder state. 1889d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org EncoderState GetEncoderState(); 1899d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1909d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Initialize after SetEncodingData in media_opt. 191accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org int Initialize(float bitrate, 192accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float user_framerate, 193accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint16_t width, 194accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint16_t height, 195accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org int num_layers); 1969d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 1979d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Update the encoder frame size. 198e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org void UpdateCodecParameters(float frame_rate, uint16_t width, uint16_t height); 1999d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2009d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Update with actual bit rate (size of the latest encoded frame) 2019d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // and frame type, after every encoded frame. 202273a414b0ec2e58fdf3b817ad8b1a02f4ce15287pbos@webrtc.org void UpdateEncodedSize(size_t encoded_size); 2039d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2049d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Update with new target bitrate, actual encoder sent rate, frame_rate, 2059d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // loss rate: every ~1 sec from SetTargetRates in media_opt. 206accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org void UpdateRates(float target_bitrate, 207accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float encoder_sent_rate, 208accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float incoming_framerate, 209accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint8_t packet_loss); 2109d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2119d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Extract ST (spatio-temporal) resolution action. 2129d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Inputs: qm: Reference to the quality modes pointer. 2139d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Output: the spatial and/or temporal scale change. 2149d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org int SelectResolution(VCMResolutionScale** qm); 2159d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 216e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org private: 217accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Set the default resolution action. 218accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org void SetDefaultAction(); 219accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 2209d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Compute rates for the selection of down-sampling action. 2219d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org void ComputeRatesForSelection(); 2229d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2239d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Compute the encoder state. 2249d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org void ComputeEncoderState(); 2259d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2269d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Return true if the action is to go back up in resolution. 2279d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org bool GoingUpResolution(); 2289d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2299d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Return true if the action is to go down in resolution. 2309d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org bool GoingDownResolution(); 2319d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2329d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Check the condition for going up in resolution by the scale factors: 2339d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // |facWidth|, |facHeight|, |facTemp|. 2349d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // |scaleFac| is a scale factor for the transition rate. 235accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org bool ConditionForGoingUp(float fac_width, 236accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float fac_height, 237accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float fac_temp, 238accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float scale_fac); 2399d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2409d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Get the bitrate threshold for the resolution action. 2419d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // The case |facWidth|=|facHeight|=|facTemp|==1 is for down-sampling action. 2429d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // |scaleFac| is a scale factor for the transition rate. 243accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float GetTransitionRate(float fac_width, 244accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float fac_height, 245accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float fac_temp, 246accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float scale_fac); 247accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 248accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Update the down-sampling state. 249accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org void UpdateDownsamplingState(UpDownAction up_down); 2509d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 251e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // Update the codec frame size and frame rate. 252e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org void UpdateCodecResolution(); 253e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org 254accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Return a state based on average target rate relative transition rate. 255accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint8_t RateClass(float transition_rate); 2569d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 257accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Adjust the action selected from the table. 2589d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org void AdjustAction(); 2599d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 260e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org // Covert 2 stages of 3/4 (=9/16) spatial decimation to 1/2. 261e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org void ConvertSpatialFractionalToWhole(); 262e22d81ce4d12c183ad7ce296f89cf10ac68858b0marpan@webrtc.org 263c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // Returns true if the new frame sizes, under the selected spatial action, 264c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org // are of even size. 265c5b392e9d6067459f7697cbf8772a5e3b0527afdmarpan@webrtc.org bool EvenFrameSize(); 266accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 267accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Insert latest down-sampling action into the history list. 268accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org void InsertLatestDownAction(); 269accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 270accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Remove the last (first element) down-sampling action from the list. 271accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org void RemoveLastDownAction(); 272accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 273accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Check constraints on the amount of down-sampling allowed. 274accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org void ConstrainAmountOfDownSampling(); 275accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 276accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // For going up in resolution: pick spatial or temporal action, 277accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // if both actions were separately selected. 278accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org void PickSpatialOrTemporal(); 279accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 2809d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Select the directional (1x2 or 2x1) spatial down-sampling action. 281accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org void SelectSpatialDirectionMode(float transition_rate); 2829d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2839d3ab61325c5ed216ea52bc829f1d8c81347459bphilipel enum { kDownActionHistorySize = 10 }; 284accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org 285accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org VCMResolutionScale* qm_; 2869d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Encoder rate control parameters. 287accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float target_bitrate_; 288accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float incoming_framerate_; 289accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float per_frame_bandwidth_; 290accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float buffer_level_; 2919d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 2929d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Data accumulated every ~1sec from MediaOpt. 293accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float sum_target_rate_; 294accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float sum_incoming_framerate_; 295accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float sum_rate_MM_; 296accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float sum_rate_MM_sgn_; 297accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float sum_packet_loss_; 2989d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Counters. 299accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint32_t frame_cnt_; 300accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint32_t frame_cnt_delta_; 301accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint32_t update_rate_cnt_; 302accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint32_t low_buffer_cnt_; 3039d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 3049d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Resolution state parameters. 305accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float state_dec_factor_spatial_; 306accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float state_dec_factor_temporal_; 3079d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 3089d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Quantities used for selection. 309accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float avg_target_rate_; 310accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float avg_incoming_framerate_; 311accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float avg_ratio_buffer_low_; 312accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float avg_rate_mismatch_; 313accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float avg_rate_mismatch_sgn_; 314accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float avg_packet_loss_; 315accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org EncoderState encoder_state_; 316accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org ResolutionAction action_; 317accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // Short history of the down-sampling actions from the Initialize() state. 318accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // This is needed for going up in resolution. Since the total amount of 319accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // down-sampling actions are constrained, the length of the list need not be 320accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org // large: i.e., (4/3) ^{kDownActionHistorySize} <= kMaxDownSample. 321accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org ResolutionAction down_action_history_[kDownActionHistorySize]; 322accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org int num_layers_; 32386548c62e91fd0cd5f3f39cd30e4e6f10b64fd79marpan@google.com}; 324470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com 3259d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org// Robustness settings class. 3269d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 3279d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.orgclass VCMQmRobustness : public VCMQmMethod { 3289d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org public: 3299d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org VCMQmRobustness(); 3309d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org ~VCMQmRobustness(); 3319d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 3329d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org virtual void Reset(); 3339d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 3349d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Adjust FEC rate based on content: every ~1 sec from SetTargetRates. 3359d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Returns an adjustment factor. 336accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float AdjustFecFactor(uint8_t code_rate_delta, 337accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float total_rate, 338accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float framerate, 33916825b1a828bb4ff40f7682040e43a239b7b8ca3pkasting@chromium.org int64_t rtt_time, 340accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint8_t packet_loss); 3419d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 3429d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Set the UEP protection on/off. 343accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org bool SetUepProtection(uint8_t code_rate_delta, 344accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float total_rate, 345accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint8_t packet_loss, 346accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org bool frame_type); 3479d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org 3489d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org private: 3499d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org // Previous state of network parameters. 350accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org float prev_total_rate_; 35116825b1a828bb4ff40f7682040e43a239b7b8ca3pkasting@chromium.org int64_t prev_rtt_time_; 352accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint8_t prev_packet_loss_; 353accf607b3e0c98cf0c16314bbf258fa1ed581aa8marpan@webrtc.org uint8_t prev_code_rate_delta_; 354470e71d3649f6cac4688e83819640b012b5d38bbniklase@google.com}; 355d900e8bea84c474696bf0219aed1353ce65ffd8epbos@webrtc.org} // namespace webrtc 3569d76b4ea5447214eb96813dde3bf7932a2745da3marpan@webrtc.org#endif // WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_ 357