1/* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#include "webrtc/modules/video_coding/utility/quality_scaler.h" 12 13#include "testing/gtest/include/gtest/gtest.h" 14 15namespace webrtc { 16namespace { 17static const int kNumSeconds = 10; 18static const int kWidth = 1920; 19static const int kHalfWidth = kWidth / 2; 20static const int kHeight = 1080; 21static const int kFramerate = 30; 22static const int kLowQp = 15; 23static const int kNormalQp = 30; 24static const int kHighQp = 40; 25static const int kMaxQp = 56; 26} // namespace 27 28class QualityScalerTest : public ::testing::Test { 29 public: 30 // Temporal and spatial resolution. 31 struct Resolution { 32 int framerate; 33 int width; 34 int height; 35 }; 36 37 protected: 38 enum ScaleDirection { 39 kKeepScaleAtHighQp, 40 kScaleDown, 41 kScaleDownAboveHighQp, 42 kScaleUp 43 }; 44 enum BadQualityMetric { kDropFrame, kReportLowQP }; 45 46 QualityScalerTest() { 47 input_frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, kHalfWidth, 48 kHalfWidth); 49 qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false); 50 qs_.ReportFramerate(kFramerate); 51 qs_.OnEncodeFrame(input_frame_); 52 } 53 54 bool TriggerScale(ScaleDirection scale_direction) { 55 qs_.OnEncodeFrame(input_frame_); 56 int initial_width = qs_.GetScaledResolution().width; 57 for (int i = 0; i < kFramerate * kNumSeconds; ++i) { 58 switch (scale_direction) { 59 case kScaleUp: 60 qs_.ReportQP(kLowQp); 61 break; 62 case kScaleDown: 63 qs_.ReportDroppedFrame(); 64 break; 65 case kKeepScaleAtHighQp: 66 qs_.ReportQP(kHighQp); 67 break; 68 case kScaleDownAboveHighQp: 69 qs_.ReportQP(kHighQp + 1); 70 break; 71 } 72 qs_.OnEncodeFrame(input_frame_); 73 if (qs_.GetScaledResolution().width != initial_width) 74 return true; 75 } 76 77 return false; 78 } 79 80 void ExpectOriginalFrame() { 81 EXPECT_EQ(&input_frame_, &qs_.GetScaledFrame(input_frame_)) 82 << "Using scaled frame instead of original input."; 83 } 84 85 void ExpectScaleUsingReportedResolution() { 86 qs_.OnEncodeFrame(input_frame_); 87 QualityScaler::Resolution res = qs_.GetScaledResolution(); 88 const VideoFrame& scaled_frame = qs_.GetScaledFrame(input_frame_); 89 EXPECT_EQ(res.width, scaled_frame.width()); 90 EXPECT_EQ(res.height, scaled_frame.height()); 91 } 92 93 void ContinuouslyDownscalesByHalfDimensionsAndBackUp(); 94 95 void DoesNotDownscaleFrameDimensions(int width, int height); 96 97 Resolution TriggerResolutionChange(BadQualityMetric dropframe_lowqp, 98 int num_second, 99 int initial_framerate); 100 101 void VerifyQualityAdaptation(int initial_framerate, 102 int seconds, 103 bool expect_spatial_resize, 104 bool expect_framerate_reduction); 105 106 void DownscaleEndsAt(int input_width, 107 int input_height, 108 int end_width, 109 int end_height); 110 111 QualityScaler qs_; 112 VideoFrame input_frame_; 113}; 114 115TEST_F(QualityScalerTest, UsesOriginalFrameInitially) { 116 ExpectOriginalFrame(); 117} 118 119TEST_F(QualityScalerTest, ReportsOriginalResolutionInitially) { 120 qs_.OnEncodeFrame(input_frame_); 121 QualityScaler::Resolution res = qs_.GetScaledResolution(); 122 EXPECT_EQ(input_frame_.width(), res.width); 123 EXPECT_EQ(input_frame_.height(), res.height); 124} 125 126TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) { 127 EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within " << kNumSeconds 128 << " seconds."; 129 QualityScaler::Resolution res = qs_.GetScaledResolution(); 130 EXPECT_LT(res.width, input_frame_.width()); 131 EXPECT_LT(res.height, input_frame_.height()); 132} 133 134TEST_F(QualityScalerTest, KeepsScaleAtHighQp) { 135 EXPECT_FALSE(TriggerScale(kKeepScaleAtHighQp)) 136 << "Downscale at high threshold which should keep scale."; 137 QualityScaler::Resolution res = qs_.GetScaledResolution(); 138 EXPECT_EQ(res.width, input_frame_.width()); 139 EXPECT_EQ(res.height, input_frame_.height()); 140} 141 142TEST_F(QualityScalerTest, DownscalesAboveHighQp) { 143 EXPECT_TRUE(TriggerScale(kScaleDownAboveHighQp)) 144 << "No downscale within " << kNumSeconds << " seconds."; 145 QualityScaler::Resolution res = qs_.GetScaledResolution(); 146 EXPECT_LT(res.width, input_frame_.width()); 147 EXPECT_LT(res.height, input_frame_.height()); 148} 149 150TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) { 151 for (int i = 0; i < kFramerate * kNumSeconds / 3; ++i) { 152 qs_.ReportQP(kNormalQp); 153 qs_.ReportDroppedFrame(); 154 qs_.ReportDroppedFrame(); 155 qs_.OnEncodeFrame(input_frame_); 156 if (qs_.GetScaledResolution().width < input_frame_.width()) 157 return; 158 } 159 160 FAIL() << "No downscale within " << kNumSeconds << " seconds."; 161} 162 163TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) { 164 for (int i = 0; i < kFramerate * kNumSeconds; ++i) { 165 qs_.ReportQP(kNormalQp); 166 qs_.OnEncodeFrame(input_frame_); 167 ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width) 168 << "Unexpected scale on half framedrop."; 169 } 170} 171 172TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) { 173 for (int i = 0; i < kFramerate * kNumSeconds / 2; ++i) { 174 qs_.ReportQP(kNormalQp); 175 qs_.OnEncodeFrame(input_frame_); 176 ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width) 177 << "Unexpected scale on half framedrop."; 178 179 qs_.ReportDroppedFrame(); 180 qs_.OnEncodeFrame(input_frame_); 181 ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width) 182 << "Unexpected scale on half framedrop."; 183 } 184} 185 186void QualityScalerTest::ContinuouslyDownscalesByHalfDimensionsAndBackUp() { 187 const int initial_min_dimension = input_frame_.width() < input_frame_.height() 188 ? input_frame_.width() 189 : input_frame_.height(); 190 int min_dimension = initial_min_dimension; 191 int current_shift = 0; 192 // Drop all frames to force-trigger downscaling. 193 while (min_dimension >= 2 * QualityScaler::kDefaultMinDownscaleDimension) { 194 EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within " 195 << kNumSeconds << " seconds."; 196 qs_.OnEncodeFrame(input_frame_); 197 QualityScaler::Resolution res = qs_.GetScaledResolution(); 198 min_dimension = res.width < res.height ? res.width : res.height; 199 ++current_shift; 200 ASSERT_EQ(input_frame_.width() >> current_shift, res.width); 201 ASSERT_EQ(input_frame_.height() >> current_shift, res.height); 202 ExpectScaleUsingReportedResolution(); 203 } 204 205 // Make sure we can scale back with good-quality frames. 206 while (min_dimension < initial_min_dimension) { 207 EXPECT_TRUE(TriggerScale(kScaleUp)) << "No upscale within " << kNumSeconds 208 << " seconds."; 209 qs_.OnEncodeFrame(input_frame_); 210 QualityScaler::Resolution res = qs_.GetScaledResolution(); 211 min_dimension = res.width < res.height ? res.width : res.height; 212 --current_shift; 213 ASSERT_EQ(input_frame_.width() >> current_shift, res.width); 214 ASSERT_EQ(input_frame_.height() >> current_shift, res.height); 215 ExpectScaleUsingReportedResolution(); 216 } 217 218 // Verify we don't start upscaling after further low use. 219 for (int i = 0; i < kFramerate * kNumSeconds; ++i) { 220 qs_.ReportQP(kLowQp); 221 ExpectOriginalFrame(); 222 } 223} 224 225TEST_F(QualityScalerTest, ContinuouslyDownscalesByHalfDimensionsAndBackUp) { 226 ContinuouslyDownscalesByHalfDimensionsAndBackUp(); 227} 228 229TEST_F(QualityScalerTest, 230 ContinuouslyDownscalesOddResolutionsByHalfDimensionsAndBackUp) { 231 const int kOddWidth = 517; 232 const int kHalfOddWidth = (kOddWidth + 1) / 2; 233 const int kOddHeight = 1239; 234 input_frame_.CreateEmptyFrame(kOddWidth, kOddHeight, kOddWidth, kHalfOddWidth, 235 kHalfOddWidth); 236 ContinuouslyDownscalesByHalfDimensionsAndBackUp(); 237} 238 239void QualityScalerTest::DoesNotDownscaleFrameDimensions(int width, int height) { 240 input_frame_.CreateEmptyFrame(width, height, width, (width + 1) / 2, 241 (width + 1) / 2); 242 243 for (int i = 0; i < kFramerate * kNumSeconds; ++i) { 244 qs_.ReportDroppedFrame(); 245 qs_.OnEncodeFrame(input_frame_); 246 ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width) 247 << "Unexpected scale of minimal-size frame."; 248 } 249} 250 251TEST_F(QualityScalerTest, DoesNotDownscaleFrom1PxWidth) { 252 DoesNotDownscaleFrameDimensions(1, kHeight); 253} 254 255TEST_F(QualityScalerTest, DoesNotDownscaleFrom1PxHeight) { 256 DoesNotDownscaleFrameDimensions(kWidth, 1); 257} 258 259TEST_F(QualityScalerTest, DoesNotDownscaleFrom1Px) { 260 DoesNotDownscaleFrameDimensions(1, 1); 261} 262 263QualityScalerTest::Resolution QualityScalerTest::TriggerResolutionChange( 264 BadQualityMetric dropframe_lowqp, 265 int num_second, 266 int initial_framerate) { 267 QualityScalerTest::Resolution res; 268 res.framerate = initial_framerate; 269 qs_.OnEncodeFrame(input_frame_); 270 res.width = qs_.GetScaledResolution().width; 271 res.height = qs_.GetScaledResolution().height; 272 for (int i = 0; i < kFramerate * num_second; ++i) { 273 switch (dropframe_lowqp) { 274 case kReportLowQP: 275 qs_.ReportQP(kLowQp); 276 break; 277 case kDropFrame: 278 qs_.ReportDroppedFrame(); 279 break; 280 } 281 qs_.OnEncodeFrame(input_frame_); 282 // Simulate the case when SetRates is called right after reducing 283 // framerate. 284 qs_.ReportFramerate(initial_framerate); 285 res.framerate = qs_.GetTargetFramerate(); 286 if (res.framerate != -1) 287 qs_.ReportFramerate(res.framerate); 288 res.width = qs_.GetScaledResolution().width; 289 res.height = qs_.GetScaledResolution().height; 290 } 291 return res; 292} 293 294void QualityScalerTest::VerifyQualityAdaptation( 295 int initial_framerate, 296 int seconds, 297 bool expect_spatial_resize, 298 bool expect_framerate_reduction) { 299 const int kDisabledBadQpThreshold = kMaxQp + 1; 300 qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, 301 kDisabledBadQpThreshold, true); 302 qs_.OnEncodeFrame(input_frame_); 303 int init_width = qs_.GetScaledResolution().width; 304 int init_height = qs_.GetScaledResolution().height; 305 306 // Test reducing framerate by dropping frame continuously. 307 QualityScalerTest::Resolution res = 308 TriggerResolutionChange(kDropFrame, seconds, initial_framerate); 309 310 if (expect_framerate_reduction) { 311 EXPECT_LT(res.framerate, initial_framerate); 312 } else { 313 // No framerate reduction, video decimator should be disabled. 314 EXPECT_EQ(-1, res.framerate); 315 } 316 317 if (expect_spatial_resize) { 318 EXPECT_LT(res.width, init_width); 319 EXPECT_LT(res.height, init_height); 320 } else { 321 EXPECT_EQ(init_width, res.width); 322 EXPECT_EQ(init_height, res.height); 323 } 324 325 // The "seconds * 1.5" is to ensure spatial resolution to recover. 326 // For example, in 10 seconds test, framerate reduction happens in the first 327 // 5 seconds from 30fps to 15fps and causes the buffer size to be half of the 328 // original one. Then it will take only 75 samples to downscale (twice in 150 329 // samples). So to recover the resolution changes, we need more than 10 330 // seconds (i.e, seconds * 1.5). This is because the framerate increases 331 // before spatial size recovers, so it will take 150 samples to recover 332 // spatial size (300 for twice). 333 res = TriggerResolutionChange(kReportLowQP, seconds * 1.5, initial_framerate); 334 EXPECT_EQ(-1, res.framerate); 335 EXPECT_EQ(init_width, res.width); 336 EXPECT_EQ(init_height, res.height); 337} 338 339// In 5 seconds test, only framerate adjusting should happen. 340TEST_F(QualityScalerTest, ChangeFramerateOnly) { 341 VerifyQualityAdaptation(kFramerate, 5, false, true); 342} 343 344// In 10 seconds test, framerate adjusting and scaling are both 345// triggered, it shows that scaling would happen after framerate 346// adjusting. 347TEST_F(QualityScalerTest, ChangeFramerateAndSpatialSize) { 348 VerifyQualityAdaptation(kFramerate, 10, true, true); 349} 350 351// When starting from a low framerate, only spatial size will be changed. 352TEST_F(QualityScalerTest, ChangeSpatialSizeOnly) { 353 qs_.ReportFramerate(kFramerate >> 1); 354 VerifyQualityAdaptation(kFramerate >> 1, 10, true, false); 355} 356 357TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) { 358 DoesNotDownscaleFrameDimensions( 359 2 * QualityScaler::kDefaultMinDownscaleDimension - 1, 1000); 360} 361 362TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsHeight) { 363 DoesNotDownscaleFrameDimensions( 364 1000, 2 * QualityScaler::kDefaultMinDownscaleDimension - 1); 365} 366 367void QualityScalerTest::DownscaleEndsAt(int input_width, 368 int input_height, 369 int end_width, 370 int end_height) { 371 // Create a frame with 2x expected end width/height to verify that we can 372 // scale down to expected end width/height. 373 input_frame_.CreateEmptyFrame(input_width, input_height, input_width, 374 (input_width + 1) / 2, (input_width + 1) / 2); 375 376 int last_width = input_width; 377 int last_height = input_height; 378 // Drop all frames to force-trigger downscaling. 379 while (true) { 380 TriggerScale(kScaleDown); 381 QualityScaler::Resolution res = qs_.GetScaledResolution(); 382 if (last_width == res.width) { 383 EXPECT_EQ(last_height, res.height); 384 EXPECT_EQ(end_width, res.width); 385 EXPECT_EQ(end_height, res.height); 386 break; 387 } 388 last_width = res.width; 389 last_height = res.height; 390 } 391} 392 393TEST_F(QualityScalerTest, DefaultDownscalesTo160x90) { 394 DownscaleEndsAt(320, 180, 160, 90); 395} 396 397TEST_F(QualityScalerTest, DefaultDownscalesTo90x160) { 398 DownscaleEndsAt(180, 320, 90, 160); 399} 400 401TEST_F(QualityScalerTest, DefaultDownscalesFrom1280x720To160x90) { 402 DownscaleEndsAt(1280, 720, 160, 90); 403} 404 405TEST_F(QualityScalerTest, DefaultDoesntDownscaleBelow160x90) { 406 DownscaleEndsAt(320 - 1, 180 - 1, 320 - 1, 180 - 1); 407} 408 409TEST_F(QualityScalerTest, DefaultDoesntDownscaleBelow90x160) { 410 DownscaleEndsAt(180 - 1, 320 - 1, 180 - 1, 320 - 1); 411} 412 413TEST_F(QualityScalerTest, RespectsMinResolutionWidth) { 414 // Should end at 200x100, as width can't go lower. 415 qs_.SetMinResolution(200, 10); 416 DownscaleEndsAt(1600, 800, 200, 100); 417} 418 419TEST_F(QualityScalerTest, RespectsMinResolutionHeight) { 420 // Should end at 100x200, as height can't go lower. 421 qs_.SetMinResolution(10, 200); 422 DownscaleEndsAt(800, 1600, 100, 200); 423} 424 425} // namespace webrtc 426