1/*
2 *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/modules/video_coding/utility/quality_scaler.h"
12
13#include "testing/gtest/include/gtest/gtest.h"
14
15namespace webrtc {
16namespace {
17static const int kNumSeconds = 10;
18static const int kWidth = 1920;
19static const int kHalfWidth = kWidth / 2;
20static const int kHeight = 1080;
21static const int kFramerate = 30;
22static const int kLowQp = 15;
23static const int kNormalQp = 30;
24static const int kHighQp = 40;
25static const int kMaxQp = 56;
26}  // namespace
27
28class QualityScalerTest : public ::testing::Test {
29 public:
30  // Temporal and spatial resolution.
31  struct Resolution {
32    int framerate;
33    int width;
34    int height;
35  };
36
37 protected:
38  enum ScaleDirection {
39    kKeepScaleAtHighQp,
40    kScaleDown,
41    kScaleDownAboveHighQp,
42    kScaleUp
43  };
44  enum BadQualityMetric { kDropFrame, kReportLowQP };
45
46  QualityScalerTest() {
47    input_frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, kHalfWidth,
48                                  kHalfWidth);
49    qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false);
50    qs_.ReportFramerate(kFramerate);
51    qs_.OnEncodeFrame(input_frame_);
52  }
53
54  bool TriggerScale(ScaleDirection scale_direction) {
55    qs_.OnEncodeFrame(input_frame_);
56    int initial_width = qs_.GetScaledResolution().width;
57    for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
58      switch (scale_direction) {
59        case kScaleUp:
60          qs_.ReportQP(kLowQp);
61          break;
62        case kScaleDown:
63          qs_.ReportDroppedFrame();
64          break;
65        case kKeepScaleAtHighQp:
66          qs_.ReportQP(kHighQp);
67          break;
68        case kScaleDownAboveHighQp:
69          qs_.ReportQP(kHighQp + 1);
70          break;
71      }
72      qs_.OnEncodeFrame(input_frame_);
73      if (qs_.GetScaledResolution().width != initial_width)
74        return true;
75    }
76
77    return false;
78  }
79
80  void ExpectOriginalFrame() {
81    EXPECT_EQ(&input_frame_, &qs_.GetScaledFrame(input_frame_))
82        << "Using scaled frame instead of original input.";
83  }
84
85  void ExpectScaleUsingReportedResolution() {
86    qs_.OnEncodeFrame(input_frame_);
87    QualityScaler::Resolution res = qs_.GetScaledResolution();
88    const VideoFrame& scaled_frame = qs_.GetScaledFrame(input_frame_);
89    EXPECT_EQ(res.width, scaled_frame.width());
90    EXPECT_EQ(res.height, scaled_frame.height());
91  }
92
93  void ContinuouslyDownscalesByHalfDimensionsAndBackUp();
94
95  void DoesNotDownscaleFrameDimensions(int width, int height);
96
97  Resolution TriggerResolutionChange(BadQualityMetric dropframe_lowqp,
98                                     int num_second,
99                                     int initial_framerate);
100
101  void VerifyQualityAdaptation(int initial_framerate,
102                               int seconds,
103                               bool expect_spatial_resize,
104                               bool expect_framerate_reduction);
105
106  void DownscaleEndsAt(int input_width,
107                       int input_height,
108                       int end_width,
109                       int end_height);
110
111  QualityScaler qs_;
112  VideoFrame input_frame_;
113};
114
115TEST_F(QualityScalerTest, UsesOriginalFrameInitially) {
116  ExpectOriginalFrame();
117}
118
119TEST_F(QualityScalerTest, ReportsOriginalResolutionInitially) {
120  qs_.OnEncodeFrame(input_frame_);
121  QualityScaler::Resolution res = qs_.GetScaledResolution();
122  EXPECT_EQ(input_frame_.width(), res.width);
123  EXPECT_EQ(input_frame_.height(), res.height);
124}
125
126TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) {
127  EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within " << kNumSeconds
128                                        << " seconds.";
129  QualityScaler::Resolution res = qs_.GetScaledResolution();
130  EXPECT_LT(res.width, input_frame_.width());
131  EXPECT_LT(res.height, input_frame_.height());
132}
133
134TEST_F(QualityScalerTest, KeepsScaleAtHighQp) {
135  EXPECT_FALSE(TriggerScale(kKeepScaleAtHighQp))
136      << "Downscale at high threshold which should keep scale.";
137  QualityScaler::Resolution res = qs_.GetScaledResolution();
138  EXPECT_EQ(res.width, input_frame_.width());
139  EXPECT_EQ(res.height, input_frame_.height());
140}
141
142TEST_F(QualityScalerTest, DownscalesAboveHighQp) {
143  EXPECT_TRUE(TriggerScale(kScaleDownAboveHighQp))
144      << "No downscale within " << kNumSeconds << " seconds.";
145  QualityScaler::Resolution res = qs_.GetScaledResolution();
146  EXPECT_LT(res.width, input_frame_.width());
147  EXPECT_LT(res.height, input_frame_.height());
148}
149
150TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
151  for (int i = 0; i < kFramerate * kNumSeconds / 3; ++i) {
152    qs_.ReportQP(kNormalQp);
153    qs_.ReportDroppedFrame();
154    qs_.ReportDroppedFrame();
155    qs_.OnEncodeFrame(input_frame_);
156    if (qs_.GetScaledResolution().width < input_frame_.width())
157      return;
158  }
159
160  FAIL() << "No downscale within " << kNumSeconds << " seconds.";
161}
162
163TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) {
164  for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
165    qs_.ReportQP(kNormalQp);
166    qs_.OnEncodeFrame(input_frame_);
167    ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
168        << "Unexpected scale on half framedrop.";
169  }
170}
171
172TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
173  for (int i = 0; i < kFramerate * kNumSeconds / 2; ++i) {
174    qs_.ReportQP(kNormalQp);
175    qs_.OnEncodeFrame(input_frame_);
176    ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
177        << "Unexpected scale on half framedrop.";
178
179    qs_.ReportDroppedFrame();
180    qs_.OnEncodeFrame(input_frame_);
181    ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
182        << "Unexpected scale on half framedrop.";
183  }
184}
185
186void QualityScalerTest::ContinuouslyDownscalesByHalfDimensionsAndBackUp() {
187  const int initial_min_dimension = input_frame_.width() < input_frame_.height()
188                                        ? input_frame_.width()
189                                        : input_frame_.height();
190  int min_dimension = initial_min_dimension;
191  int current_shift = 0;
192  // Drop all frames to force-trigger downscaling.
193  while (min_dimension >= 2 * QualityScaler::kDefaultMinDownscaleDimension) {
194    EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within "
195                                          << kNumSeconds << " seconds.";
196    qs_.OnEncodeFrame(input_frame_);
197    QualityScaler::Resolution res = qs_.GetScaledResolution();
198    min_dimension = res.width < res.height ? res.width : res.height;
199    ++current_shift;
200    ASSERT_EQ(input_frame_.width() >> current_shift, res.width);
201    ASSERT_EQ(input_frame_.height() >> current_shift, res.height);
202    ExpectScaleUsingReportedResolution();
203  }
204
205  // Make sure we can scale back with good-quality frames.
206  while (min_dimension < initial_min_dimension) {
207    EXPECT_TRUE(TriggerScale(kScaleUp)) << "No upscale within " << kNumSeconds
208                                        << " seconds.";
209    qs_.OnEncodeFrame(input_frame_);
210    QualityScaler::Resolution res = qs_.GetScaledResolution();
211    min_dimension = res.width < res.height ? res.width : res.height;
212    --current_shift;
213    ASSERT_EQ(input_frame_.width() >> current_shift, res.width);
214    ASSERT_EQ(input_frame_.height() >> current_shift, res.height);
215    ExpectScaleUsingReportedResolution();
216  }
217
218  // Verify we don't start upscaling after further low use.
219  for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
220    qs_.ReportQP(kLowQp);
221    ExpectOriginalFrame();
222  }
223}
224
225TEST_F(QualityScalerTest, ContinuouslyDownscalesByHalfDimensionsAndBackUp) {
226  ContinuouslyDownscalesByHalfDimensionsAndBackUp();
227}
228
229TEST_F(QualityScalerTest,
230       ContinuouslyDownscalesOddResolutionsByHalfDimensionsAndBackUp) {
231  const int kOddWidth = 517;
232  const int kHalfOddWidth = (kOddWidth + 1) / 2;
233  const int kOddHeight = 1239;
234  input_frame_.CreateEmptyFrame(kOddWidth, kOddHeight, kOddWidth, kHalfOddWidth,
235                                kHalfOddWidth);
236  ContinuouslyDownscalesByHalfDimensionsAndBackUp();
237}
238
239void QualityScalerTest::DoesNotDownscaleFrameDimensions(int width, int height) {
240  input_frame_.CreateEmptyFrame(width, height, width, (width + 1) / 2,
241                                (width + 1) / 2);
242
243  for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
244    qs_.ReportDroppedFrame();
245    qs_.OnEncodeFrame(input_frame_);
246    ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
247        << "Unexpected scale of minimal-size frame.";
248  }
249}
250
251TEST_F(QualityScalerTest, DoesNotDownscaleFrom1PxWidth) {
252  DoesNotDownscaleFrameDimensions(1, kHeight);
253}
254
255TEST_F(QualityScalerTest, DoesNotDownscaleFrom1PxHeight) {
256  DoesNotDownscaleFrameDimensions(kWidth, 1);
257}
258
259TEST_F(QualityScalerTest, DoesNotDownscaleFrom1Px) {
260  DoesNotDownscaleFrameDimensions(1, 1);
261}
262
263QualityScalerTest::Resolution QualityScalerTest::TriggerResolutionChange(
264    BadQualityMetric dropframe_lowqp,
265    int num_second,
266    int initial_framerate) {
267  QualityScalerTest::Resolution res;
268  res.framerate = initial_framerate;
269  qs_.OnEncodeFrame(input_frame_);
270  res.width = qs_.GetScaledResolution().width;
271  res.height = qs_.GetScaledResolution().height;
272  for (int i = 0; i < kFramerate * num_second; ++i) {
273    switch (dropframe_lowqp) {
274      case kReportLowQP:
275        qs_.ReportQP(kLowQp);
276        break;
277      case kDropFrame:
278        qs_.ReportDroppedFrame();
279        break;
280    }
281    qs_.OnEncodeFrame(input_frame_);
282    // Simulate the case when SetRates is called right after reducing
283    // framerate.
284    qs_.ReportFramerate(initial_framerate);
285    res.framerate = qs_.GetTargetFramerate();
286    if (res.framerate != -1)
287      qs_.ReportFramerate(res.framerate);
288    res.width = qs_.GetScaledResolution().width;
289    res.height = qs_.GetScaledResolution().height;
290  }
291  return res;
292}
293
294void QualityScalerTest::VerifyQualityAdaptation(
295    int initial_framerate,
296    int seconds,
297    bool expect_spatial_resize,
298    bool expect_framerate_reduction) {
299  const int kDisabledBadQpThreshold = kMaxQp + 1;
300  qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator,
301           kDisabledBadQpThreshold, true);
302  qs_.OnEncodeFrame(input_frame_);
303  int init_width = qs_.GetScaledResolution().width;
304  int init_height = qs_.GetScaledResolution().height;
305
306  // Test reducing framerate by dropping frame continuously.
307  QualityScalerTest::Resolution res =
308      TriggerResolutionChange(kDropFrame, seconds, initial_framerate);
309
310  if (expect_framerate_reduction) {
311    EXPECT_LT(res.framerate, initial_framerate);
312  } else {
313    // No framerate reduction, video decimator should be disabled.
314    EXPECT_EQ(-1, res.framerate);
315  }
316
317  if (expect_spatial_resize) {
318    EXPECT_LT(res.width, init_width);
319    EXPECT_LT(res.height, init_height);
320  } else {
321    EXPECT_EQ(init_width, res.width);
322    EXPECT_EQ(init_height, res.height);
323  }
324
325  // The "seconds * 1.5" is to ensure spatial resolution to recover.
326  // For example, in 10 seconds test, framerate reduction happens in the first
327  // 5 seconds from 30fps to 15fps and causes the buffer size to be half of the
328  // original one. Then it will take only 75 samples to downscale (twice in 150
329  // samples). So to recover the resolution changes, we need more than 10
330  // seconds (i.e, seconds * 1.5). This is because the framerate increases
331  // before spatial size recovers, so it will take 150 samples to recover
332  // spatial size (300 for twice).
333  res = TriggerResolutionChange(kReportLowQP, seconds * 1.5, initial_framerate);
334  EXPECT_EQ(-1, res.framerate);
335  EXPECT_EQ(init_width, res.width);
336  EXPECT_EQ(init_height, res.height);
337}
338
339// In 5 seconds test, only framerate adjusting should happen.
340TEST_F(QualityScalerTest, ChangeFramerateOnly) {
341  VerifyQualityAdaptation(kFramerate, 5, false, true);
342}
343
344// In 10 seconds test, framerate adjusting and scaling are both
345// triggered, it shows that scaling would happen after framerate
346// adjusting.
347TEST_F(QualityScalerTest, ChangeFramerateAndSpatialSize) {
348  VerifyQualityAdaptation(kFramerate, 10, true, true);
349}
350
351// When starting from a low framerate, only spatial size will be changed.
352TEST_F(QualityScalerTest, ChangeSpatialSizeOnly) {
353  qs_.ReportFramerate(kFramerate >> 1);
354  VerifyQualityAdaptation(kFramerate >> 1, 10, true, false);
355}
356
357TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) {
358  DoesNotDownscaleFrameDimensions(
359      2 * QualityScaler::kDefaultMinDownscaleDimension - 1, 1000);
360}
361
362TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsHeight) {
363  DoesNotDownscaleFrameDimensions(
364      1000, 2 * QualityScaler::kDefaultMinDownscaleDimension - 1);
365}
366
367void QualityScalerTest::DownscaleEndsAt(int input_width,
368                                        int input_height,
369                                        int end_width,
370                                        int end_height) {
371  // Create a frame with 2x expected end width/height to verify that we can
372  // scale down to expected end width/height.
373  input_frame_.CreateEmptyFrame(input_width, input_height, input_width,
374                                (input_width + 1) / 2, (input_width + 1) / 2);
375
376  int last_width = input_width;
377  int last_height = input_height;
378  // Drop all frames to force-trigger downscaling.
379  while (true) {
380    TriggerScale(kScaleDown);
381    QualityScaler::Resolution res = qs_.GetScaledResolution();
382    if (last_width == res.width) {
383      EXPECT_EQ(last_height, res.height);
384      EXPECT_EQ(end_width, res.width);
385      EXPECT_EQ(end_height, res.height);
386      break;
387    }
388    last_width = res.width;
389    last_height = res.height;
390  }
391}
392
393TEST_F(QualityScalerTest, DefaultDownscalesTo160x90) {
394  DownscaleEndsAt(320, 180, 160, 90);
395}
396
397TEST_F(QualityScalerTest, DefaultDownscalesTo90x160) {
398  DownscaleEndsAt(180, 320, 90, 160);
399}
400
401TEST_F(QualityScalerTest, DefaultDownscalesFrom1280x720To160x90) {
402  DownscaleEndsAt(1280, 720, 160, 90);
403}
404
405TEST_F(QualityScalerTest, DefaultDoesntDownscaleBelow160x90) {
406  DownscaleEndsAt(320 - 1, 180 - 1, 320 - 1, 180 - 1);
407}
408
409TEST_F(QualityScalerTest, DefaultDoesntDownscaleBelow90x160) {
410  DownscaleEndsAt(180 - 1, 320 - 1, 180 - 1, 320 - 1);
411}
412
413TEST_F(QualityScalerTest, RespectsMinResolutionWidth) {
414  // Should end at 200x100, as width can't go lower.
415  qs_.SetMinResolution(200, 10);
416  DownscaleEndsAt(1600, 800, 200, 100);
417}
418
419TEST_F(QualityScalerTest, RespectsMinResolutionHeight) {
420  // Should end at 100x200, as height can't go lower.
421  qs_.SetMinResolution(10, 200);
422  DownscaleEndsAt(800, 1600, 100, 200);
423}
424
425}  // namespace webrtc
426