1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include <math.h>
12
13#include "testing/gtest/include/gtest/gtest.h"
14
15#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
16#include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h"
17#include "webrtc/modules/video_coding/codecs/test/videoprocessor.h"
18#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
19#include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
20#include "webrtc/modules/video_coding/main/interface/video_coding.h"
21#include "webrtc/test/testsupport/fileutils.h"
22#include "webrtc/test/testsupport/frame_reader.h"
23#include "webrtc/test/testsupport/frame_writer.h"
24#include "webrtc/test/testsupport/gtest_disable.h"
25#include "webrtc/test/testsupport/metrics/video_metrics.h"
26#include "webrtc/test/testsupport/packet_reader.h"
27#include "webrtc/typedefs.h"
28
29namespace webrtc {
30
31// Maximum number of rate updates (i.e., calls to encoder to change bitrate
32// and/or frame rate) for the current tests.
33const int kMaxNumRateUpdates = 3;
34
35const int kPercTargetvsActualMismatch = 20;
36const int kBaseKeyFrameInterval = 3000;
37
38// Codec and network settings.
39struct CodecConfigPars {
40  float packet_loss;
41  int num_temporal_layers;
42  int key_frame_interval;
43  bool error_concealment_on;
44  bool denoising_on;
45  bool frame_dropper_on;
46  bool spatial_resize_on;
47};
48
49// Quality metrics.
50struct QualityMetrics {
51  double minimum_avg_psnr;
52  double minimum_min_psnr;
53  double minimum_avg_ssim;
54  double minimum_min_ssim;
55};
56
57// The sequence of bitrate and frame rate changes for the encoder, the frame
58// number where the changes are made, and the total number of frames for the
59// test.
60struct RateProfile {
61  int target_bit_rate[kMaxNumRateUpdates];
62  int input_frame_rate[kMaxNumRateUpdates];
63  int frame_index_rate_update[kMaxNumRateUpdates + 1];
64  int num_frames;
65};
66
67// Metrics for the rate control. The rate mismatch metrics are defined as
68// percentages.|max_time_hit_target| is defined as number of frames, after a
69// rate update is made to the encoder, for the encoder to reach within
70// |kPercTargetvsActualMismatch| of new target rate. The metrics are defined for
71// each rate update sequence.
72struct RateControlMetrics {
73  int max_num_dropped_frames;
74  int max_key_frame_size_mismatch;
75  int max_delta_frame_size_mismatch;
76  int max_encoding_rate_mismatch;
77  int max_time_hit_target;
78  int num_spatial_resizes;
79};
80
81
82// Sequence used is foreman (CIF): may be better to use VGA for resize test.
83const int kCIFWidth = 352;
84const int kCIFHeight = 288;
85const int kNbrFramesShort = 100;  // Some tests are run for shorter sequence.
86const int kNbrFramesLong = 299;
87
88// Parameters from VP8 wrapper, which control target size of key frames.
89const float kInitialBufferSize = 0.5f;
90const float kOptimalBufferSize = 0.6f;
91const float kScaleKeyFrameSize = 0.5f;
92
93// Integration test for video processor. Encodes+decodes a clip and
94// writes it to the output directory. After completion, quality metrics
95// (PSNR and SSIM) and rate control metrics are computed to verify that the
96// quality and encoder response is acceptable. The rate control tests allow us
97// to verify the behavior for changing bitrate, changing frame rate, frame
98// dropping/spatial resize, and temporal layers. The limits for the rate
99// control metrics are set to be fairly conservative, so failure should only
100// happen when some significant regression or breakdown occurs.
101class VideoProcessorIntegrationTest: public testing::Test {
102 protected:
103  VideoEncoder* encoder_;
104  VideoDecoder* decoder_;
105  webrtc::test::FrameReader* frame_reader_;
106  webrtc::test::FrameWriter* frame_writer_;
107  webrtc::test::PacketReader packet_reader_;
108  webrtc::test::PacketManipulator* packet_manipulator_;
109  webrtc::test::Stats stats_;
110  webrtc::test::TestConfig config_;
111  VideoCodec codec_settings_;
112  webrtc::test::VideoProcessor* processor_;
113
114  // Quantities defined/updated for every encoder rate update.
115  // Some quantities defined per temporal layer (at most 3 layers in this test).
116  int num_frames_per_update_[3];
117  float sum_frame_size_mismatch_[3];
118  float sum_encoded_frame_size_[3];
119  float encoding_bitrate_[3];
120  float per_frame_bandwidth_[3];
121  float bit_rate_layer_[3];
122  float frame_rate_layer_[3];
123  int num_frames_total_;
124  float sum_encoded_frame_size_total_;
125  float encoding_bitrate_total_;
126  float perc_encoding_rate_mismatch_;
127  int num_frames_to_hit_target_;
128  bool encoding_rate_within_target_;
129  int bit_rate_;
130  int frame_rate_;
131  int layer_;
132  float target_size_key_frame_initial_;
133  float target_size_key_frame_;
134  float sum_key_frame_size_mismatch_;
135  int num_key_frames_;
136  float start_bitrate_;
137
138  // Codec and network settings.
139  float packet_loss_;
140  int num_temporal_layers_;
141  int key_frame_interval_;
142  bool error_concealment_on_;
143  bool denoising_on_;
144  bool frame_dropper_on_;
145  bool spatial_resize_on_;
146
147
148  VideoProcessorIntegrationTest() {}
149  virtual ~VideoProcessorIntegrationTest() {}
150
151  void SetUpCodecConfig() {
152    encoder_ = VP8Encoder::Create();
153    decoder_ = VP8Decoder::Create();
154
155    // CIF is currently used for all tests below.
156    // Setup the TestConfig struct for processing of a clip in CIF resolution.
157    config_.input_filename =
158        webrtc::test::ResourcePath("foreman_cif", "yuv");
159
160    // Generate an output filename in a safe way.
161    config_.output_filename = webrtc::test::TempFilename(
162        webrtc::test::OutputPath(), "videoprocessor_integrationtest");
163    config_.frame_length_in_bytes = CalcBufferSize(kI420,
164                                                   kCIFWidth, kCIFHeight);
165    config_.verbose = false;
166    // Only allow encoder/decoder to use single core, for predictability.
167    config_.use_single_core = true;
168    // Key frame interval and packet loss are set for each test.
169    config_.keyframe_interval = key_frame_interval_;
170    config_.networking_config.packet_loss_probability = packet_loss_;
171
172    // Get a codec configuration struct and configure it.
173    VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_);
174    config_.codec_settings = &codec_settings_;
175    config_.codec_settings->startBitrate = start_bitrate_;
176    config_.codec_settings->width = kCIFWidth;
177    config_.codec_settings->height = kCIFHeight;
178    // These features may be set depending on the test.
179    config_.codec_settings->codecSpecific.VP8.errorConcealmentOn =
180        error_concealment_on_;
181    config_.codec_settings->codecSpecific.VP8.denoisingOn =
182        denoising_on_;
183    config_.codec_settings->codecSpecific.VP8.numberOfTemporalLayers =
184        num_temporal_layers_;
185    config_.codec_settings->codecSpecific.VP8.frameDroppingOn =
186        frame_dropper_on_;
187    config_.codec_settings->codecSpecific.VP8.automaticResizeOn =
188        spatial_resize_on_;
189    config_.codec_settings->codecSpecific.VP8.keyFrameInterval =
190        kBaseKeyFrameInterval;
191
192    frame_reader_ =
193        new webrtc::test::FrameReaderImpl(config_.input_filename,
194                                          config_.frame_length_in_bytes);
195    frame_writer_ =
196        new webrtc::test::FrameWriterImpl(config_.output_filename,
197                                          config_.frame_length_in_bytes);
198    ASSERT_TRUE(frame_reader_->Init());
199    ASSERT_TRUE(frame_writer_->Init());
200
201    packet_manipulator_ = new webrtc::test::PacketManipulatorImpl(
202        &packet_reader_, config_.networking_config, config_.verbose);
203    processor_ = new webrtc::test::VideoProcessorImpl(encoder_, decoder_,
204                                                      frame_reader_,
205                                                      frame_writer_,
206                                                      packet_manipulator_,
207                                                      config_, &stats_);
208    ASSERT_TRUE(processor_->Init());
209  }
210
211  // Reset quantities after each encoder update, update the target
212  // per-frame bandwidth.
213  void ResetRateControlMetrics(int num_frames) {
214    for (int i = 0; i < num_temporal_layers_; i++) {
215      num_frames_per_update_[i] = 0;
216      sum_frame_size_mismatch_[i] = 0.0f;
217      sum_encoded_frame_size_[i] = 0.0f;
218      encoding_bitrate_[i] = 0.0f;
219      // Update layer per-frame-bandwidth.
220      per_frame_bandwidth_[i] = static_cast<float>(bit_rate_layer_[i]) /
221             static_cast<float>(frame_rate_layer_[i]);
222    }
223    // Set maximum size of key frames, following setting in the VP8 wrapper.
224    float max_key_size = kScaleKeyFrameSize * kOptimalBufferSize * frame_rate_;
225    // We don't know exact target size of the key frames (except for first one),
226    // but the minimum in libvpx is ~|3 * per_frame_bandwidth| and maximum is
227    // set by |max_key_size_  * per_frame_bandwidth|. Take middle point/average
228    // as reference for mismatch. Note key frames always correspond to base
229    // layer frame in this test.
230    target_size_key_frame_ = 0.5 * (3 + max_key_size) * per_frame_bandwidth_[0];
231    num_frames_total_ = 0;
232    sum_encoded_frame_size_total_ = 0.0f;
233    encoding_bitrate_total_ = 0.0f;
234    perc_encoding_rate_mismatch_ = 0.0f;
235    num_frames_to_hit_target_ = num_frames;
236    encoding_rate_within_target_ = false;
237    sum_key_frame_size_mismatch_ = 0.0;
238    num_key_frames_ = 0;
239  }
240
241  // For every encoded frame, update the rate control metrics.
242  void UpdateRateControlMetrics(int frame_num, VideoFrameType frame_type) {
243    int encoded_frame_size = processor_->EncodedFrameSize();
244    float encoded_size_kbits = encoded_frame_size * 8.0f / 1000.0f;
245    // Update layer data.
246    // Update rate mismatch relative to per-frame bandwidth for delta frames.
247    if (frame_type == kDeltaFrame) {
248      // TODO(marpan): Should we count dropped (zero size) frames in mismatch?
249      sum_frame_size_mismatch_[layer_] += fabs(encoded_size_kbits -
250                                               per_frame_bandwidth_[layer_]) /
251                                               per_frame_bandwidth_[layer_];
252    } else {
253      float target_size = (frame_num == 1) ? target_size_key_frame_initial_ :
254          target_size_key_frame_;
255      sum_key_frame_size_mismatch_ += fabs(encoded_size_kbits - target_size) /
256          target_size;
257      num_key_frames_ += 1;
258    }
259    sum_encoded_frame_size_[layer_] += encoded_size_kbits;
260    // Encoding bitrate per layer: from the start of the update/run to the
261    // current frame.
262    encoding_bitrate_[layer_] = sum_encoded_frame_size_[layer_] *
263        frame_rate_layer_[layer_] /
264        num_frames_per_update_[layer_];
265    // Total encoding rate: from the start of the update/run to current frame.
266    sum_encoded_frame_size_total_ += encoded_size_kbits;
267    encoding_bitrate_total_ = sum_encoded_frame_size_total_ * frame_rate_ /
268        num_frames_total_;
269    perc_encoding_rate_mismatch_ =  100 * fabs(encoding_bitrate_total_ -
270                                               bit_rate_) / bit_rate_;
271    if (perc_encoding_rate_mismatch_ < kPercTargetvsActualMismatch &&
272        !encoding_rate_within_target_) {
273      num_frames_to_hit_target_ = num_frames_total_;
274      encoding_rate_within_target_ = true;
275    }
276  }
277
278  // Verify expected behavior of rate control and print out data.
279  void VerifyRateControl(int update_index,
280                         int max_key_frame_size_mismatch,
281                         int max_delta_frame_size_mismatch,
282                         int max_encoding_rate_mismatch,
283                         int max_time_hit_target,
284                         int max_num_dropped_frames,
285                         int num_spatial_resizes) {
286    int num_dropped_frames = processor_->NumberDroppedFrames();
287    int num_resize_actions = processor_->NumberSpatialResizes();
288    printf("For update #: %d,\n "
289        " Target Bitrate: %d,\n"
290        " Encoding bitrate: %f,\n"
291        " Frame rate: %d \n",
292        update_index, bit_rate_, encoding_bitrate_total_, frame_rate_);
293    printf(" Number of frames to approach target rate = %d, \n"
294           " Number of dropped frames = %d, \n"
295           " Number of spatial resizes = %d, \n",
296           num_frames_to_hit_target_, num_dropped_frames, num_resize_actions);
297    EXPECT_LE(perc_encoding_rate_mismatch_, max_encoding_rate_mismatch);
298    if (num_key_frames_ > 0) {
299      int perc_key_frame_size_mismatch = 100 * sum_key_frame_size_mismatch_ /
300              num_key_frames_;
301      printf(" Number of Key frames: %d \n"
302             " Key frame rate mismatch: %d \n",
303             num_key_frames_, perc_key_frame_size_mismatch);
304      EXPECT_LE(perc_key_frame_size_mismatch, max_key_frame_size_mismatch);
305    }
306    printf("\n");
307    printf("Rates statistics for Layer data \n");
308    for (int i = 0; i < num_temporal_layers_ ; i++) {
309      printf("Layer #%d \n", i);
310      int perc_frame_size_mismatch = 100 * sum_frame_size_mismatch_[i] /
311        num_frames_per_update_[i];
312      int perc_encoding_rate_mismatch = 100 * fabs(encoding_bitrate_[i] -
313                                                   bit_rate_layer_[i]) /
314                                                   bit_rate_layer_[i];
315      printf(" Target Layer Bit rate: %f \n"
316          " Layer frame rate: %f, \n"
317          " Layer per frame bandwidth: %f, \n"
318          " Layer Encoding bit rate: %f, \n"
319          " Layer Percent frame size mismatch: %d,  \n"
320          " Layer Percent encoding rate mismatch = %d, \n"
321          " Number of frame processed per layer = %d \n",
322          bit_rate_layer_[i], frame_rate_layer_[i], per_frame_bandwidth_[i],
323          encoding_bitrate_[i], perc_frame_size_mismatch,
324          perc_encoding_rate_mismatch, num_frames_per_update_[i]);
325      EXPECT_LE(perc_frame_size_mismatch, max_delta_frame_size_mismatch);
326      EXPECT_LE(perc_encoding_rate_mismatch, max_encoding_rate_mismatch);
327    }
328    printf("\n");
329    EXPECT_LE(num_frames_to_hit_target_, max_time_hit_target);
330    EXPECT_LE(num_dropped_frames, max_num_dropped_frames);
331    EXPECT_EQ(num_resize_actions, num_spatial_resizes);
332  }
333
334  // Layer index corresponding to frame number, for up to 3 layers.
335  void LayerIndexForFrame(int frame_number) {
336    if (num_temporal_layers_ == 1) {
337      layer_ = 0;
338    } else if (num_temporal_layers_ == 2) {
339        // layer 0:  0     2     4 ...
340        // layer 1:     1     3
341        if (frame_number % 2 == 0) {
342          layer_ = 0;
343        } else {
344          layer_ = 1;
345        }
346    } else if (num_temporal_layers_ == 3) {
347      // layer 0:  0            4            8 ...
348      // layer 1:        2            6
349      // layer 2:     1      3     5      7
350      if (frame_number % 4 == 0) {
351        layer_ = 0;
352      } else if ((frame_number + 2) % 4 == 0) {
353        layer_ = 1;
354      } else if ((frame_number + 1) % 2 == 0) {
355        layer_ = 2;
356      }
357    } else {
358      assert(false);  // Only up to 3 layers.
359    }
360  }
361
362  // Set the bitrate and frame rate per layer, for up to 3 layers.
363  void SetLayerRates() {
364    assert(num_temporal_layers_<= 3);
365    for (int i = 0; i < num_temporal_layers_; i++) {
366      float bit_rate_ratio =
367          kVp8LayerRateAlloction[num_temporal_layers_ - 1][i];
368      if (i > 0) {
369        float bit_rate_delta_ratio = kVp8LayerRateAlloction
370            [num_temporal_layers_ - 1][i] -
371            kVp8LayerRateAlloction[num_temporal_layers_ - 1][i - 1];
372        bit_rate_layer_[i] = bit_rate_ * bit_rate_delta_ratio;
373      } else {
374        bit_rate_layer_[i] = bit_rate_ * bit_rate_ratio;
375      }
376      frame_rate_layer_[i] = frame_rate_ / static_cast<float>(
377          1 << (num_temporal_layers_ - 1));
378    }
379    if (num_temporal_layers_ == 3) {
380      frame_rate_layer_[2] = frame_rate_ / 2.0f;
381    }
382  }
383
384  VideoFrameType FrameType(int frame_number) {
385    if (frame_number == 0 || ((frame_number) % key_frame_interval_ == 0 &&
386        key_frame_interval_ > 0)) {
387      return kKeyFrame;
388    } else {
389      return kDeltaFrame;
390    }
391  }
392
393  void TearDown() {
394    delete processor_;
395    delete packet_manipulator_;
396    delete frame_writer_;
397    delete frame_reader_;
398    delete decoder_;
399    delete encoder_;
400  }
401
402  // Processes all frames in the clip and verifies the result.
403  void ProcessFramesAndVerify(QualityMetrics quality_metrics,
404                              RateProfile rate_profile,
405                              CodecConfigPars process,
406                              RateControlMetrics* rc_metrics) {
407    // Codec/config settings.
408    start_bitrate_ = rate_profile.target_bit_rate[0];
409    packet_loss_ = process.packet_loss;
410    key_frame_interval_ = process.key_frame_interval;
411    num_temporal_layers_ = process.num_temporal_layers;
412    error_concealment_on_ = process.error_concealment_on;
413    denoising_on_ = process.denoising_on;
414    frame_dropper_on_ = process.frame_dropper_on;
415    spatial_resize_on_ = process.spatial_resize_on;
416    SetUpCodecConfig();
417    // Update the layers and the codec with the initial rates.
418    bit_rate_ =  rate_profile.target_bit_rate[0];
419    frame_rate_ = rate_profile.input_frame_rate[0];
420    SetLayerRates();
421    // Set the initial target size for key frame.
422    target_size_key_frame_initial_ = 0.5 * kInitialBufferSize *
423        bit_rate_layer_[0];
424    processor_->SetRates(bit_rate_, frame_rate_);
425    // Process each frame, up to |num_frames|.
426    int num_frames = rate_profile.num_frames;
427    int update_index = 0;
428    ResetRateControlMetrics(
429        rate_profile.frame_index_rate_update[update_index + 1]);
430    int frame_number = 0;
431    VideoFrameType frame_type = kDeltaFrame;
432    while (processor_->ProcessFrame(frame_number) &&
433        frame_number < num_frames) {
434      // Get the layer index for the frame |frame_number|.
435      LayerIndexForFrame(frame_number);
436      frame_type = FrameType(frame_number);
437      // Counter for whole sequence run.
438      ++frame_number;
439      // Counters for each rate update.
440      ++num_frames_per_update_[layer_];
441      ++num_frames_total_;
442      UpdateRateControlMetrics(frame_number, frame_type);
443      // If we hit another/next update, verify stats for current state and
444      // update layers and codec with new rates.
445      if (frame_number ==
446          rate_profile.frame_index_rate_update[update_index + 1]) {
447        VerifyRateControl(
448            update_index,
449            rc_metrics[update_index].max_key_frame_size_mismatch,
450            rc_metrics[update_index].max_delta_frame_size_mismatch,
451            rc_metrics[update_index].max_encoding_rate_mismatch,
452            rc_metrics[update_index].max_time_hit_target,
453            rc_metrics[update_index].max_num_dropped_frames,
454            rc_metrics[update_index].num_spatial_resizes);
455        // Update layer rates and the codec with new rates.
456        ++update_index;
457        bit_rate_ =  rate_profile.target_bit_rate[update_index];
458        frame_rate_ = rate_profile.input_frame_rate[update_index];
459        SetLayerRates();
460        ResetRateControlMetrics(rate_profile.
461                                frame_index_rate_update[update_index + 1]);
462        processor_->SetRates(bit_rate_, frame_rate_);
463      }
464    }
465    VerifyRateControl(
466        update_index,
467        rc_metrics[update_index].max_key_frame_size_mismatch,
468        rc_metrics[update_index].max_delta_frame_size_mismatch,
469        rc_metrics[update_index].max_encoding_rate_mismatch,
470        rc_metrics[update_index].max_time_hit_target,
471        rc_metrics[update_index].max_num_dropped_frames,
472        rc_metrics[update_index].num_spatial_resizes);
473    EXPECT_EQ(num_frames, frame_number);
474    EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size()));
475
476    // Release encoder and decoder to make sure they have finished processing:
477    EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
478    EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
479    // Close the files before we start using them for SSIM/PSNR calculations.
480    frame_reader_->Close();
481    frame_writer_->Close();
482
483    // TODO(marpan): should compute these quality metrics per SetRates update.
484    webrtc::test::QualityMetricsResult psnr_result, ssim_result;
485    EXPECT_EQ(0, webrtc::test::I420MetricsFromFiles(
486        config_.input_filename.c_str(),
487        config_.output_filename.c_str(),
488        config_.codec_settings->width,
489        config_.codec_settings->height,
490        &psnr_result,
491        &ssim_result));
492    printf("PSNR avg: %f, min: %f    SSIM avg: %f, min: %f\n",
493           psnr_result.average, psnr_result.min,
494           ssim_result.average, ssim_result.min);
495    stats_.PrintSummary();
496    EXPECT_GT(psnr_result.average, quality_metrics.minimum_avg_psnr);
497    EXPECT_GT(psnr_result.min, quality_metrics.minimum_min_psnr);
498    EXPECT_GT(ssim_result.average, quality_metrics.minimum_avg_ssim);
499    EXPECT_GT(ssim_result.min, quality_metrics.minimum_min_ssim);
500    if (!remove(config_.output_filename.c_str())) {
501      fprintf(stderr, "Failed to remove temporary file!");
502    }
503  }
504};
505
506void SetRateProfilePars(RateProfile* rate_profile,
507                        int update_index,
508                        int bit_rate,
509                        int frame_rate,
510                        int frame_index_rate_update) {
511  rate_profile->target_bit_rate[update_index] = bit_rate;
512  rate_profile->input_frame_rate[update_index] = frame_rate;
513  rate_profile->frame_index_rate_update[update_index] = frame_index_rate_update;
514}
515
516void SetCodecParameters(CodecConfigPars* process_settings,
517                        float packet_loss,
518                        int key_frame_interval,
519                        int num_temporal_layers,
520                        bool error_concealment_on,
521                        bool denoising_on,
522                        bool frame_dropper_on,
523                        bool spatial_resize_on) {
524  process_settings->packet_loss = packet_loss;
525  process_settings->key_frame_interval =  key_frame_interval;
526  process_settings->num_temporal_layers = num_temporal_layers,
527  process_settings->error_concealment_on = error_concealment_on;
528  process_settings->denoising_on = denoising_on;
529  process_settings->frame_dropper_on = frame_dropper_on;
530  process_settings->spatial_resize_on = spatial_resize_on;
531}
532
533void SetQualityMetrics(QualityMetrics* quality_metrics,
534                       double minimum_avg_psnr,
535                       double minimum_min_psnr,
536                       double minimum_avg_ssim,
537                       double minimum_min_ssim) {
538  quality_metrics->minimum_avg_psnr = minimum_avg_psnr;
539  quality_metrics->minimum_min_psnr = minimum_min_psnr;
540  quality_metrics->minimum_avg_ssim = minimum_avg_ssim;
541  quality_metrics->minimum_min_ssim = minimum_min_ssim;
542}
543
544void SetRateControlMetrics(RateControlMetrics* rc_metrics,
545                           int update_index,
546                           int max_num_dropped_frames,
547                           int max_key_frame_size_mismatch,
548                           int max_delta_frame_size_mismatch,
549                           int max_encoding_rate_mismatch,
550                           int max_time_hit_target,
551                           int num_spatial_resizes) {
552  rc_metrics[update_index].max_num_dropped_frames = max_num_dropped_frames;
553  rc_metrics[update_index].max_key_frame_size_mismatch =
554      max_key_frame_size_mismatch;
555  rc_metrics[update_index].max_delta_frame_size_mismatch =
556      max_delta_frame_size_mismatch;
557  rc_metrics[update_index].max_encoding_rate_mismatch =
558      max_encoding_rate_mismatch;
559  rc_metrics[update_index].max_time_hit_target = max_time_hit_target;
560  rc_metrics[update_index].num_spatial_resizes = num_spatial_resizes;
561}
562
563// Run with no packet loss and fixed bitrate. Quality should be very high.
564// One key frame (first frame only) in sequence. Setting |key_frame_interval|
565// to -1 below means no periodic key frames in test.
566TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) {
567  // Bitrate and frame rate profile.
568  RateProfile rate_profile;
569  SetRateProfilePars(&rate_profile, 0, 500, 30, 0);
570  rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1;
571  rate_profile.num_frames = kNbrFramesShort;
572  // Codec/network settings.
573  CodecConfigPars process_settings;
574  SetCodecParameters(&process_settings, 0.0f, -1, 1, false, true, true, false);
575  // Metrics for expected quality.
576  QualityMetrics quality_metrics;
577  SetQualityMetrics(&quality_metrics, 34.95, 33.0, 0.90, 0.89);
578  // Metrics for rate control.
579  RateControlMetrics rc_metrics[1];
580  SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0);
581  ProcessFramesAndVerify(quality_metrics,
582                         rate_profile,
583                         process_settings,
584                         rc_metrics);
585}
586
587// Run with 5% packet loss and fixed bitrate. Quality should be a bit lower.
588// One key frame (first frame only) in sequence.
589TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) {
590  // Bitrate and frame rate profile.
591  RateProfile rate_profile;
592  SetRateProfilePars(&rate_profile, 0, 500, 30, 0);
593  rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1;
594  rate_profile.num_frames = kNbrFramesShort;
595  // Codec/network settings.
596  CodecConfigPars process_settings;
597  SetCodecParameters(&process_settings, 0.05f, -1, 1, false, true, true, false);
598  // Metrics for expected quality.
599  QualityMetrics quality_metrics;
600  SetQualityMetrics(&quality_metrics, 20.0, 16.0, 0.60, 0.40);
601  // Metrics for rate control.
602  RateControlMetrics rc_metrics[1];
603  SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0);
604  ProcessFramesAndVerify(quality_metrics,
605                         rate_profile,
606                         process_settings,
607                         rc_metrics);
608}
609
610// Run with 10% packet loss and fixed bitrate. Quality should be even lower.
611// One key frame (first frame only) in sequence.
612TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) {
613  // Bitrate and frame rate profile.
614  RateProfile rate_profile;
615  SetRateProfilePars(&rate_profile, 0, 500, 30, 0);
616  rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1;
617  rate_profile.num_frames = kNbrFramesShort;
618  // Codec/network settings.
619  CodecConfigPars process_settings;
620  SetCodecParameters(&process_settings, 0.1f, -1, 1, false, true, true, false);
621  // Metrics for expected quality.
622  QualityMetrics quality_metrics;
623  SetQualityMetrics(&quality_metrics, 19.0, 16.0, 0.50, 0.35);
624  // Metrics for rate control.
625  RateControlMetrics rc_metrics[1];
626  SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0);
627  ProcessFramesAndVerify(quality_metrics,
628                         rate_profile,
629                         process_settings,
630                         rc_metrics);
631}
632
633// The tests below are currently disabled for Android. For ARM, the encoder
634// uses |cpu_speed| = 12, as opposed to default |cpu_speed| <= 6 for x86,
635// which leads to significantly different quality. The quality and rate control
636// settings in the tests below are defined for encoder speed setting
637// |cpu_speed| <= ~6. A number of settings would need to be significantly
638// modified for the |cpu_speed| = 12 case. For now, keep the tests below
639// disabled on Android. Some quality parameter in the above test has been
640// adjusted to also pass for |cpu_speed| <= 12.
641
642// Run with no packet loss, with varying bitrate (3 rate updates):
643// low to high to medium. Check that quality and encoder response to the new
644// target rate/per-frame bandwidth (for each rate update) is within limits.
645// One key frame (first frame only) in sequence.
646TEST_F(VideoProcessorIntegrationTest,
647       DISABLED_ON_ANDROID(ProcessNoLossChangeBitRate)) {
648  // Bitrate and frame rate profile.
649  RateProfile rate_profile;
650  SetRateProfilePars(&rate_profile, 0, 200, 30, 0);
651  SetRateProfilePars(&rate_profile, 1, 800, 30, 100);
652  SetRateProfilePars(&rate_profile, 2, 500, 30, 200);
653  rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1;
654  rate_profile.num_frames = kNbrFramesLong;
655  // Codec/network settings.
656  CodecConfigPars process_settings;
657  SetCodecParameters(&process_settings, 0.0f, -1, 1, false, true, true, false);
658  // Metrics for expected quality.
659  QualityMetrics quality_metrics;
660  SetQualityMetrics(&quality_metrics, 34.0, 32.0, 0.85, 0.80);
661  // Metrics for rate control.
662  RateControlMetrics rc_metrics[3];
663  SetRateControlMetrics(rc_metrics, 0, 0, 45, 20, 10, 15, 0);
664  SetRateControlMetrics(rc_metrics, 1, 0, 0, 25, 20, 10, 0);
665  SetRateControlMetrics(rc_metrics, 2, 0, 0, 25, 15, 10, 0);
666  ProcessFramesAndVerify(quality_metrics,
667                         rate_profile,
668                         process_settings,
669                         rc_metrics);
670}
671
672// Run with no packet loss, with an update (decrease) in frame rate.
673// Lower frame rate means higher per-frame-bandwidth, so easier to encode.
674// At the bitrate in this test, this means better rate control after the
675// update(s) to lower frame rate. So expect less frame drops, and max values
676// for the rate control metrics can be lower. One key frame (first frame only).
677// Note: quality after update should be higher but we currently compute quality
678// metrics avergaed over whole sequence run.
679TEST_F(VideoProcessorIntegrationTest,
680       DISABLED_ON_ANDROID(ProcessNoLossChangeFrameRateFrameDrop)) {
681  config_.networking_config.packet_loss_probability = 0;
682  // Bitrate and frame rate profile.
683  RateProfile rate_profile;
684  SetRateProfilePars(&rate_profile, 0, 80, 24, 0);
685  SetRateProfilePars(&rate_profile, 1, 80, 15, 100);
686  SetRateProfilePars(&rate_profile, 2, 80, 10, 200);
687  rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1;
688  rate_profile.num_frames = kNbrFramesLong;
689  // Codec/network settings.
690  CodecConfigPars process_settings;
691  SetCodecParameters(&process_settings, 0.0f, -1, 1, false, true, true, false);
692  // Metrics for expected quality.
693  QualityMetrics quality_metrics;
694  SetQualityMetrics(&quality_metrics, 31.0, 22.0, 0.80, 0.65);
695  // Metrics for rate control.
696  RateControlMetrics rc_metrics[3];
697  SetRateControlMetrics(rc_metrics, 0, 40, 20, 75, 15, 60, 0);
698  SetRateControlMetrics(rc_metrics, 1, 10, 0, 25, 10, 35, 0);
699  SetRateControlMetrics(rc_metrics, 2, 0, 0, 20, 10, 15, 0);
700  ProcessFramesAndVerify(quality_metrics,
701                         rate_profile,
702                         process_settings,
703                         rc_metrics);
704}
705
706// Run with no packet loss, at low bitrate. During this time we should've
707// resized once.
708TEST_F(VideoProcessorIntegrationTest,
709       DISABLED_ON_ANDROID(ProcessNoLossSpatialResizeFrameDrop)) {
710  config_.networking_config.packet_loss_probability = 0;
711  // Bitrate and frame rate profile.
712  RateProfile rate_profile;
713  SetRateProfilePars(&rate_profile, 0, 50, 30, 0);
714  rate_profile.frame_index_rate_update[1] = kNbrFramesLong + 1;
715  rate_profile.num_frames = kNbrFramesLong;
716  // Codec/network settings.
717  CodecConfigPars process_settings;
718  SetCodecParameters(
719      &process_settings, 0.0f, kNbrFramesLong, 1, false, true, true, true);
720  // Metrics for expected quality.
721  QualityMetrics quality_metrics;
722  SetQualityMetrics(&quality_metrics, 25.0, 15.0, 0.70, 0.40);
723  // Metrics for rate control.
724  RateControlMetrics rc_metrics[1];
725  SetRateControlMetrics(rc_metrics, 0, 160, 60, 120, 20, 70, 1);
726  ProcessFramesAndVerify(quality_metrics,
727                         rate_profile,
728                         process_settings,
729                         rc_metrics);
730}
731
732// Run with no packet loss, with 3 temporal layers, with a rate update in the
733// middle of the sequence. The max values for the frame size mismatch and
734// encoding rate mismatch are applied to each layer.
735// No dropped frames in this test, and internal spatial resizer is off.
736// One key frame (first frame only) in sequence, so no spatial resizing.
737TEST_F(VideoProcessorIntegrationTest,
738       DISABLED_ON_ANDROID(ProcessNoLossTemporalLayers)) {
739  config_.networking_config.packet_loss_probability = 0;
740  // Bitrate and frame rate profile.
741  RateProfile rate_profile;
742  SetRateProfilePars(&rate_profile, 0, 200, 30, 0);
743  SetRateProfilePars(&rate_profile, 1, 400, 30, 150);
744  rate_profile.frame_index_rate_update[2] = kNbrFramesLong + 1;
745  rate_profile.num_frames = kNbrFramesLong;
746  // Codec/network settings.
747  CodecConfigPars process_settings;
748  SetCodecParameters(&process_settings, 0.0f, -1, 3, false, true, true, false);
749  // Metrics for expected quality.
750  QualityMetrics quality_metrics;
751  SetQualityMetrics(&quality_metrics, 32.5, 30.0, 0.85, 0.80);
752  // Metrics for rate control.
753  RateControlMetrics rc_metrics[2];
754  SetRateControlMetrics(rc_metrics, 0, 0, 20, 30, 10, 10, 0);
755  SetRateControlMetrics(rc_metrics, 1, 0, 0, 30, 15, 10, 0);
756  ProcessFramesAndVerify(quality_metrics,
757                         rate_profile,
758                         process_settings,
759                         rc_metrics);
760}
761}  // namespace webrtc
762