1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include <sstream>
12#include <string>
13#include <vector>
14
15#include "testing/gtest/include/gtest/gtest.h"
16#include "webrtc/test/testsupport/fileutils.h"
17#include "webrtc/test/testsupport/metrics/video_metrics.h"
18#include "webrtc/test/testsupport/perf_test.h"
19#include "webrtc/video_engine/test/auto_test/interface/vie_autotest.h"
20#include "webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h"
21#include "webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.h"
22#include "webrtc/video_engine/test/libvietest/include/tb_external_transport.h"
23#include "webrtc/video_engine/test/libvietest/include/vie_to_file_renderer.h"
24
25namespace {
26
27// The input file must be QCIF since I420 gets scaled to that in the tests
28// (it is so bandwidth-heavy we have no choice). Our comparison algorithms
29// wouldn't like scaling, so this will work when we compare with the original.
30const int kInputWidth = 176;
31const int kInputHeight = 144;
32
33class ViEVideoVerificationTest : public testing::Test {
34 protected:
35  void SetUp() {
36    input_file_ = webrtc::test::ResourcePath("paris_qcif", "yuv");
37    local_file_renderer_ = NULL;
38    remote_file_renderer_ = NULL;
39  }
40
41  void InitializeFileRenderers() {
42    local_file_renderer_ = new ViEToFileRenderer();
43    remote_file_renderer_ = new ViEToFileRenderer();
44    SetUpLocalFileRenderer(local_file_renderer_);
45    SetUpRemoteFileRenderer(remote_file_renderer_);
46  }
47
48  void SetUpLocalFileRenderer(ViEToFileRenderer* file_renderer) {
49    SetUpFileRenderer(file_renderer, "-local-preview.yuv");
50  }
51
52  void SetUpRemoteFileRenderer(ViEToFileRenderer* file_renderer) {
53    SetUpFileRenderer(file_renderer, "-remote.yuv");
54  }
55
56  // Must be called manually inside the tests.
57  void StopRenderers() {
58    local_file_renderer_->StopRendering();
59    remote_file_renderer_->StopRendering();
60  }
61
62  void CompareFiles(const std::string& reference_file,
63                    const std::string& test_file,
64                    double* psnr_result, double *ssim_result) {
65    webrtc::test::QualityMetricsResult psnr;
66    int error = I420PSNRFromFiles(reference_file.c_str(), test_file.c_str(),
67                                  kInputWidth, kInputHeight, &psnr);
68
69    EXPECT_EQ(0, error) << "PSNR routine failed - output files missing?";
70    *psnr_result = psnr.average;
71
72    webrtc::test::QualityMetricsResult ssim;
73    error = I420SSIMFromFiles(reference_file.c_str(), test_file.c_str(),
74                              kInputWidth, kInputHeight, &ssim);
75    EXPECT_EQ(0, error) << "SSIM routine failed - output files missing?";
76    *ssim_result = ssim.average;
77
78    ViETest::Log("Results: PSNR is %f (dB; 48 is max), "
79                 "SSIM is %f (1 is perfect)",
80                 psnr.average, ssim.average);
81  }
82
83  // Note: must call AFTER CompareFiles.
84  void TearDownFileRenderers() {
85    TearDownFileRenderer(local_file_renderer_);
86    TearDownFileRenderer(remote_file_renderer_);
87  }
88
89  std::string input_file_;
90  ViEToFileRenderer* local_file_renderer_;
91  ViEToFileRenderer* remote_file_renderer_;
92  ViEFileBasedComparisonTests tests_;
93
94 private:
95  void SetUpFileRenderer(ViEToFileRenderer* file_renderer,
96                         const std::string& suffix) {
97    std::string output_path = ViETest::GetResultOutputPath();
98    std::string filename = "render_output" + suffix;
99
100    if (!file_renderer->PrepareForRendering(output_path, filename)) {
101      FAIL() << "Could not open output file " << filename <<
102          " for writing.";
103    }
104  }
105
106  void TearDownFileRenderer(ViEToFileRenderer* file_renderer) {
107      assert(file_renderer);
108      bool test_failed = ::testing::UnitTest::GetInstance()->
109          current_test_info()->result()->Failed();
110      if (test_failed) {
111        // Leave the files for analysis if the test failed.
112        file_renderer->SaveOutputFile("failed-");
113      }
114      delete file_renderer;
115    }
116};
117
118class ParameterizedFullStackTest : public ViEVideoVerificationTest,
119                                   public ::testing::WithParamInterface<int> {
120 public:
121  static const int kNumFullStackInstances = 4;
122
123 protected:
124  struct TestParameters {
125    NetworkParameters network;
126    std::string file_name;
127    int width;
128    int height;
129    int bitrate;
130    double avg_psnr_threshold;
131    double avg_ssim_threshold;
132    ProtectionMethod protection_method;
133    std::string test_label;
134  };
135
136  void SetUp() {
137    for (int i = 0; i < kNumFullStackInstances; ++i) {
138      parameter_table_[i].file_name = webrtc::test::ResourcePath("foreman_cif",
139                                                                 "yuv");
140      parameter_table_[i].width = 352;
141      parameter_table_[i].height = 288;
142    }
143    int i = 0;
144    parameter_table_[i].protection_method = kNack;
145    // Uniform loss => Setting burst length to -1.
146    parameter_table_[i].network.loss_model = kUniformLoss;
147    parameter_table_[i].network.packet_loss_rate = 0;
148    parameter_table_[i].network.burst_length = -1;
149    parameter_table_[i].network.mean_one_way_delay = 0;
150    parameter_table_[i].network.std_dev_one_way_delay = 0;
151    parameter_table_[i].bitrate = 300;
152    // TODO(holmer): Enable for Win and Mac when the file rendering has been
153    // moved to a separate thread.
154#ifdef WEBRTC_LINUX
155    parameter_table_[i].avg_psnr_threshold = 35;
156    parameter_table_[i].avg_ssim_threshold = 0.96;
157#else
158    parameter_table_[i].avg_psnr_threshold = 0;
159    parameter_table_[i].avg_ssim_threshold = 0.0;
160#endif
161    parameter_table_[i].test_label = "net_delay_0_0_plr_0";
162    ++i;
163    parameter_table_[i].protection_method = kNack;
164    parameter_table_[i].network.loss_model = kUniformLoss;
165    parameter_table_[i].network.packet_loss_rate = 5;
166    parameter_table_[i].network.burst_length = -1;
167    parameter_table_[i].network.mean_one_way_delay = 50;
168    parameter_table_[i].network.std_dev_one_way_delay = 5;
169    parameter_table_[i].bitrate = 300;
170    // TODO(holmer): Enable for Win and Mac when the file rendering has been
171    // moved to a separate thread.
172#ifdef WEBRTC_LINUX
173    parameter_table_[i].avg_psnr_threshold = 35;
174    parameter_table_[i].avg_ssim_threshold = 0.96;
175#else
176    parameter_table_[i].avg_psnr_threshold = 0;
177    parameter_table_[i].avg_ssim_threshold = 0.0;
178#endif
179    parameter_table_[i].test_label = "net_delay_50_5_plr_5";
180    ++i;
181    parameter_table_[i].protection_method = kNack;
182    parameter_table_[i].network.loss_model = kUniformLoss;
183    parameter_table_[i].network.packet_loss_rate = 0;
184    parameter_table_[i].network.burst_length = -1;
185    parameter_table_[i].network.mean_one_way_delay = 100;
186    parameter_table_[i].network.std_dev_one_way_delay = 10;
187    parameter_table_[i].bitrate = 300;
188    // TODO(holmer): Enable for Win and Mac when the file rendering has been
189    // moved to a separate thread.
190#ifdef WEBRTC_LINUX
191    parameter_table_[i].avg_psnr_threshold = 35;
192    parameter_table_[i].avg_ssim_threshold = 0.96;
193#else
194    parameter_table_[i].avg_psnr_threshold = 0;
195    parameter_table_[i].avg_ssim_threshold = 0.0;
196#endif
197    parameter_table_[i].test_label = "net_delay_100_10_plr_0";
198    ++i;
199    parameter_table_[i].protection_method = kNack;
200    parameter_table_[i].network.loss_model = kGilbertElliotLoss;
201    parameter_table_[i].network.packet_loss_rate = 5;
202    parameter_table_[i].network.burst_length = 3;
203    parameter_table_[i].network.mean_one_way_delay = 100;
204    parameter_table_[i].network.std_dev_one_way_delay = 10;
205    parameter_table_[i].bitrate = 300;
206    // Thresholds disabled for now. This is being run mainly to get a graph.
207    parameter_table_[i].avg_psnr_threshold = 0;
208    parameter_table_[i].avg_ssim_threshold = 0.0;
209    parameter_table_[i].test_label = "net_delay_100_10_plr_5_gilbert_elliot";
210
211    ASSERT_EQ(kNumFullStackInstances - 1, i);
212  }
213
214  TestParameters parameter_table_[kNumFullStackInstances];
215};
216
217TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
218  // I420 is lossless, so the I420 test should obviously get perfect results -
219  // the local preview and remote output files should be bit-exact. This test
220  // runs on external transport to ensure we do not drop packets.
221  // However, it's hard to make 100% stringent requirements on the video engine
222  // since for instance the jitter buffer has non-deterministic elements. If it
223  // breaks five times in a row though, you probably introduced a bug.
224  const double kReasonablePsnr = webrtc::test::kMetricsPerfectPSNR - 2.0f;
225  const double kReasonableSsim = 0.99f;
226  const int kNumAttempts = 5;
227  for (int attempt = 0; attempt < kNumAttempts; ++attempt) {
228    InitializeFileRenderers();
229    ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight,
230                                     local_file_renderer_,
231                                     remote_file_renderer_));
232    std::string remote_file = remote_file_renderer_->GetFullOutputPath();
233    std::string local_preview = local_file_renderer_->GetFullOutputPath();
234    StopRenderers();
235
236    double actual_psnr = 0;
237    double actual_ssim = 0;
238    CompareFiles(local_preview, remote_file, &actual_psnr, &actual_ssim);
239
240    TearDownFileRenderers();
241
242    if (actual_psnr > kReasonablePsnr && actual_ssim > kReasonableSsim) {
243      // Test successful.
244      return;
245    } else {
246      ViETest::Log("Retrying; attempt %d of %d.", attempt + 1, kNumAttempts);
247    }
248  }
249
250  FAIL() << "Failed to achieve near-perfect PSNR and SSIM results after " <<
251      kNumAttempts << " attempts.";
252}
253
254// Runs a whole stack processing with tracking of which frames are dropped
255// in the encoder. Tests show that they start at the same frame, which is
256// the important thing when doing frame-to-frame comparison with PSNR/SSIM.
257TEST_P(ParameterizedFullStackTest, RunsFullStackWithoutErrors)  {
258  // Using CIF here since it's a more common resolution than QCIF, and higher
259  // resolutions shouldn't be a problem for a test using VP8.
260  input_file_ = parameter_table_[GetParam()].file_name;
261  FrameDropDetector detector;
262  local_file_renderer_ = new ViEToFileRenderer();
263  remote_file_renderer_ = new FrameDropMonitoringRemoteFileRenderer(&detector);
264  SetUpLocalFileRenderer(local_file_renderer_);
265  SetUpRemoteFileRenderer(remote_file_renderer_);
266
267  // Set a low bit rate so the encoder budget will be tight, causing it to drop
268  // frames every now and then.
269  const int kBitRateKbps = parameter_table_[GetParam()].bitrate;
270  const NetworkParameters network = parameter_table_[GetParam()].network;
271  int width = parameter_table_[GetParam()].width;
272  int height = parameter_table_[GetParam()].height;
273  ProtectionMethod protection_method =
274      parameter_table_[GetParam()].protection_method;
275  ViETest::Log("Bit rate     : %5d kbps", kBitRateKbps);
276  ViETest::Log("Packet loss  : %5d %%", network.packet_loss_rate);
277  ViETest::Log("Network delay: mean=%dms std dev=%d ms",
278               network.mean_one_way_delay, network.std_dev_one_way_delay);
279  tests_.TestFullStack(input_file_, width, height, kBitRateKbps,
280                       protection_method, network, local_file_renderer_,
281                       remote_file_renderer_, &detector);
282  const std::string reference_file = local_file_renderer_->GetFullOutputPath();
283  const std::string output_file = remote_file_renderer_->GetFullOutputPath();
284  StopRenderers();
285
286  detector.CalculateResults();
287  detector.PrintReport(parameter_table_[GetParam()].test_label);
288
289  if (detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered) >
290      detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded)) {
291    detector.PrintDebugDump();
292  }
293
294  ASSERT_GE(detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered),
295      detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded))
296      << "The number of dropped frames on the decode and render steps are not "
297      "equal. This may be because we have a major problem in the buffers of "
298      "the ViEToFileRenderer?";
299
300  // We may have dropped frames during the processing, which means the output
301  // file does not contain all the frames that are present in the input file.
302  // To make the quality measurement correct, we must adjust the output file to
303  // that by copying the last successful frame into the place where the dropped
304  // frame would be, for all dropped frames.
305  const int frame_length_in_bytes = 3 * width * height / 2;
306  ViETest::Log("Frame length: %d bytes", frame_length_in_bytes);
307  std::vector<Frame*> all_frames = detector.GetAllFrames();
308  FixOutputFileForComparison(output_file, frame_length_in_bytes, all_frames);
309
310  // Verify all sent frames are present in the output file.
311  size_t output_file_size = webrtc::test::GetFileSize(output_file);
312  EXPECT_EQ(all_frames.size(), output_file_size / frame_length_in_bytes)
313      << "The output file size is incorrect. It should be equal to the number "
314      "of frames multiplied by the frame size. This will likely affect "
315      "PSNR/SSIM calculations in a bad way.";
316
317  TearDownFileRenderers();
318
319  // We are running on a lower bitrate here so we need to settle for somewhat
320  // lower PSNR and SSIM values.
321  double actual_psnr = 0;
322  double actual_ssim = 0;
323  CompareFiles(reference_file, output_file, &actual_psnr, &actual_ssim);
324
325  const double kExpectedMinimumPSNR =
326      parameter_table_[GetParam()].avg_psnr_threshold;
327  const double kExpectedMinimumSSIM =
328      parameter_table_[GetParam()].avg_ssim_threshold;
329
330  EXPECT_GE(actual_psnr, kExpectedMinimumPSNR);
331  EXPECT_GE(actual_ssim, kExpectedMinimumSSIM);
332
333  std::stringstream ss;
334  ss << std::setprecision(3) << std::fixed << actual_psnr;
335  webrtc::test::PrintResult(
336      "psnr", "", parameter_table_[GetParam()].test_label,
337      ss.str(), "dB", false);
338
339  ss.str("");
340  ss << std::setprecision(3) << std::fixed << actual_ssim;
341  webrtc::test::PrintResult(
342      "ssim", "", parameter_table_[GetParam()].test_label,
343      ss.str(), "", false);
344}
345
346INSTANTIATE_TEST_CASE_P(FullStackTests, ParameterizedFullStackTest,
347    ::testing::Range(0, ParameterizedFullStackTest::kNumFullStackInstances));
348
349}  // namespace
350