1/*
2 *  Copyright (c) 2013 The WebM project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "third_party/googletest/src/include/gtest/gtest.h"
12#include "test/codec_factory.h"
13#include "test/encode_test_driver.h"
14#include "test/i420_video_source.h"
15#include "test/util.h"
16
17namespace {
18
19const int kMaxErrorFrames = 12;
20const int kMaxDroppableFrames = 12;
21
22class ErrorResilienceTestLarge : public ::libvpx_test::EncoderTest,
23    public ::libvpx_test::CodecTestWithParam<libvpx_test::TestMode> {
24 protected:
25  ErrorResilienceTestLarge()
26      : EncoderTest(GET_PARAM(0)),
27        psnr_(0.0),
28        nframes_(0),
29        mismatch_psnr_(0.0),
30        mismatch_nframes_(0),
31        encoding_mode_(GET_PARAM(1)) {
32    Reset();
33  }
34
35  virtual ~ErrorResilienceTestLarge() {}
36
37  void Reset() {
38    error_nframes_ = 0;
39    droppable_nframes_ = 0;
40  }
41
42  virtual void SetUp() {
43    InitializeConfig();
44    SetMode(encoding_mode_);
45  }
46
47  virtual void BeginPassHook(unsigned int /*pass*/) {
48    psnr_ = 0.0;
49    nframes_ = 0;
50    mismatch_psnr_ = 0.0;
51    mismatch_nframes_ = 0;
52  }
53
54  virtual void PSNRPktHook(const vpx_codec_cx_pkt_t *pkt) {
55    psnr_ += pkt->data.psnr.psnr[0];
56    nframes_++;
57  }
58
59  virtual void PreEncodeFrameHook(libvpx_test::VideoSource *video) {
60    frame_flags_ &= ~(VP8_EFLAG_NO_UPD_LAST |
61                      VP8_EFLAG_NO_UPD_GF |
62                      VP8_EFLAG_NO_UPD_ARF);
63    if (droppable_nframes_ > 0 &&
64        (cfg_.g_pass == VPX_RC_LAST_PASS || cfg_.g_pass == VPX_RC_ONE_PASS)) {
65      for (unsigned int i = 0; i < droppable_nframes_; ++i) {
66        if (droppable_frames_[i] == video->frame()) {
67          std::cout << "             Encoding droppable frame: "
68                    << droppable_frames_[i] << "\n";
69          frame_flags_ |= (VP8_EFLAG_NO_UPD_LAST |
70                           VP8_EFLAG_NO_UPD_GF |
71                           VP8_EFLAG_NO_UPD_ARF);
72          return;
73        }
74      }
75    }
76  }
77
78  double GetAveragePsnr() const {
79    if (nframes_)
80      return psnr_ / nframes_;
81    return 0.0;
82  }
83
84  double GetAverageMismatchPsnr() const {
85    if (mismatch_nframes_)
86      return mismatch_psnr_ / mismatch_nframes_;
87    return 0.0;
88  }
89
90  virtual bool DoDecode() const {
91    if (error_nframes_ > 0 &&
92        (cfg_.g_pass == VPX_RC_LAST_PASS || cfg_.g_pass == VPX_RC_ONE_PASS)) {
93      for (unsigned int i = 0; i < error_nframes_; ++i) {
94        if (error_frames_[i] == nframes_ - 1) {
95          std::cout << "             Skipping decoding frame: "
96                    << error_frames_[i] << "\n";
97          return 0;
98        }
99      }
100    }
101    return 1;
102  }
103
104  virtual void MismatchHook(const vpx_image_t *img1,
105                            const vpx_image_t *img2) {
106    double mismatch_psnr = compute_psnr(img1, img2);
107    mismatch_psnr_ += mismatch_psnr;
108    ++mismatch_nframes_;
109    // std::cout << "Mismatch frame psnr: " << mismatch_psnr << "\n";
110  }
111
112  void SetErrorFrames(int num, unsigned int *list) {
113    if (num > kMaxErrorFrames)
114      num = kMaxErrorFrames;
115    else if (num < 0)
116      num = 0;
117    error_nframes_ = num;
118    for (unsigned int i = 0; i < error_nframes_; ++i)
119      error_frames_[i] = list[i];
120  }
121
122  void SetDroppableFrames(int num, unsigned int *list) {
123    if (num > kMaxDroppableFrames)
124      num = kMaxDroppableFrames;
125    else if (num < 0)
126      num = 0;
127    droppable_nframes_ = num;
128    for (unsigned int i = 0; i < droppable_nframes_; ++i)
129      droppable_frames_[i] = list[i];
130  }
131
132  unsigned int GetMismatchFrames() {
133    return mismatch_nframes_;
134  }
135
136 private:
137  double psnr_;
138  unsigned int nframes_;
139  unsigned int error_nframes_;
140  unsigned int droppable_nframes_;
141  double mismatch_psnr_;
142  unsigned int mismatch_nframes_;
143  unsigned int error_frames_[kMaxErrorFrames];
144  unsigned int droppable_frames_[kMaxDroppableFrames];
145  libvpx_test::TestMode encoding_mode_;
146};
147
148TEST_P(ErrorResilienceTestLarge, OnVersusOff) {
149  const vpx_rational timebase = { 33333333, 1000000000 };
150  cfg_.g_timebase = timebase;
151  cfg_.rc_target_bitrate = 2000;
152  cfg_.g_lag_in_frames = 10;
153
154  init_flags_ = VPX_CODEC_USE_PSNR;
155
156  libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
157                                     timebase.den, timebase.num, 0, 30);
158
159  // Error resilient mode OFF.
160  cfg_.g_error_resilient = 0;
161  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
162  const double psnr_resilience_off = GetAveragePsnr();
163  EXPECT_GT(psnr_resilience_off, 25.0);
164
165  // Error resilient mode ON.
166  cfg_.g_error_resilient = 1;
167  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
168  const double psnr_resilience_on = GetAveragePsnr();
169  EXPECT_GT(psnr_resilience_on, 25.0);
170
171  // Test that turning on error resilient mode hurts by 10% at most.
172  if (psnr_resilience_off > 0.0) {
173    const double psnr_ratio = psnr_resilience_on / psnr_resilience_off;
174    EXPECT_GE(psnr_ratio, 0.9);
175    EXPECT_LE(psnr_ratio, 1.1);
176  }
177}
178
179// Check for successful decoding and no encoder/decoder mismatch
180// if we lose (i.e., drop before decoding) a set of droppable
181// frames (i.e., frames that don't update any reference buffers).
182// Check both isolated and consecutive loss.
183TEST_P(ErrorResilienceTestLarge, DropFramesWithoutRecovery) {
184  const vpx_rational timebase = { 33333333, 1000000000 };
185  cfg_.g_timebase = timebase;
186  cfg_.rc_target_bitrate = 500;
187  // FIXME(debargha): Fix this to work for any lag.
188  // Currently this test only works for lag = 0
189  cfg_.g_lag_in_frames = 0;
190
191  init_flags_ = VPX_CODEC_USE_PSNR;
192
193  libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
194                                     timebase.den, timebase.num, 0, 40);
195
196  // Error resilient mode ON.
197  cfg_.g_error_resilient = 1;
198  cfg_.kf_mode = VPX_KF_DISABLED;
199
200  // Set an arbitrary set of error frames same as droppable frames.
201  // In addition to isolated loss/drop, add a long consecutive series
202  // (of size 9) of dropped frames.
203  unsigned int num_droppable_frames = 11;
204  unsigned int droppable_frame_list[] = {5, 16, 22, 23, 24, 25, 26, 27, 28,
205                                         29, 30};
206  SetDroppableFrames(num_droppable_frames, droppable_frame_list);
207  SetErrorFrames(num_droppable_frames, droppable_frame_list);
208  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
209  // Test that no mismatches have been found
210  std::cout << "             Mismatch frames: "
211            << GetMismatchFrames() << "\n";
212  EXPECT_EQ(GetMismatchFrames(), (unsigned int) 0);
213
214  // Reset previously set of error/droppable frames.
215  Reset();
216
217#if 0
218  // TODO(jkoleszar): This test is disabled for the time being as too
219  // sensitive. It's not clear how to set a reasonable threshold for
220  // this behavior.
221
222  // Now set an arbitrary set of error frames that are non-droppable
223  unsigned int num_error_frames = 3;
224  unsigned int error_frame_list[] = {3, 10, 20};
225  SetErrorFrames(num_error_frames, error_frame_list);
226  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
227
228  // Test that dropping an arbitrary set of inter frames does not hurt too much
229  // Note the Average Mismatch PSNR is the average of the PSNR between
230  // decoded frame and encoder's version of the same frame for all frames
231  // with mismatch.
232  const double psnr_resilience_mismatch = GetAverageMismatchPsnr();
233  std::cout << "             Mismatch PSNR: "
234            << psnr_resilience_mismatch << "\n";
235  EXPECT_GT(psnr_resilience_mismatch, 20.0);
236#endif
237}
238
239VP8_INSTANTIATE_TEST_CASE(ErrorResilienceTestLarge, ONE_PASS_TEST_MODES);
240VP9_INSTANTIATE_TEST_CASE(ErrorResilienceTestLarge, ONE_PASS_TEST_MODES);
241
242}  // namespace
243