1/*
2 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10#include <stdio.h>
11
12#include <deque>
13#include <map>
14
15#include "testing/gtest/include/gtest/gtest.h"
16
17#include "webrtc/base/thread_annotations.h"
18#include "webrtc/call.h"
19#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
20#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
21#include "webrtc/system_wrappers/interface/clock.h"
22#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
23#include "webrtc/system_wrappers/interface/event_wrapper.h"
24#include "webrtc/system_wrappers/interface/scoped_ptr.h"
25#include "webrtc/system_wrappers/interface/sleep.h"
26#include "webrtc/test/call_test.h"
27#include "webrtc/test/direct_transport.h"
28#include "webrtc/test/encoder_settings.h"
29#include "webrtc/test/fake_encoder.h"
30#include "webrtc/test/frame_generator_capturer.h"
31#include "webrtc/test/statistics.h"
32#include "webrtc/test/testsupport/fileutils.h"
33#include "webrtc/typedefs.h"
34
35namespace webrtc {
36
37static const int kFullStackTestDurationSecs = 60;
38
39struct FullStackTestParams {
40  const char* test_label;
41  struct {
42    const char* name;
43    size_t width, height;
44    int fps;
45  } clip;
46  int min_bitrate_bps;
47  int target_bitrate_bps;
48  int max_bitrate_bps;
49  double avg_psnr_threshold;
50  double avg_ssim_threshold;
51  FakeNetworkPipe::Config link;
52};
53
54class FullStackTest : public test::CallTest {
55 protected:
56  void RunTest(const FullStackTestParams& params);
57};
58
59class VideoAnalyzer : public PacketReceiver,
60                      public newapi::Transport,
61                      public VideoRenderer,
62                      public VideoSendStreamInput {
63 public:
64  VideoAnalyzer(VideoSendStreamInput* input,
65                Transport* transport,
66                const char* test_label,
67                double avg_psnr_threshold,
68                double avg_ssim_threshold,
69                int duration_frames)
70      : input_(input),
71        transport_(transport),
72        receiver_(NULL),
73        test_label_(test_label),
74        frames_left_(duration_frames),
75        dropped_frames_(0),
76        last_render_time_(0),
77        rtp_timestamp_delta_(0),
78        crit_(CriticalSectionWrapper::CreateCriticalSection()),
79        first_send_frame_(NULL),
80        avg_psnr_threshold_(avg_psnr_threshold),
81        avg_ssim_threshold_(avg_ssim_threshold),
82        comparison_lock_(CriticalSectionWrapper::CreateCriticalSection()),
83        comparison_thread_(ThreadWrapper::CreateThread(&FrameComparisonThread,
84                                                       this)),
85        done_(EventWrapper::Create()) {
86    unsigned int id;
87    EXPECT_TRUE(comparison_thread_->Start(id));
88  }
89
90  ~VideoAnalyzer() {
91    EXPECT_TRUE(comparison_thread_->Stop());
92
93    while (!frames_.empty()) {
94      delete frames_.back();
95      frames_.pop_back();
96    }
97    while (!frame_pool_.empty()) {
98      delete frame_pool_.back();
99      frame_pool_.pop_back();
100    }
101  }
102
103  virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; }
104
105  virtual DeliveryStatus DeliverPacket(const uint8_t* packet,
106                                       size_t length) OVERRIDE {
107    scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
108    RTPHeader header;
109    parser->Parse(packet, length, &header);
110    {
111      CriticalSectionScoped lock(crit_.get());
112      recv_times_[header.timestamp - rtp_timestamp_delta_] =
113          Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
114    }
115
116    return receiver_->DeliverPacket(packet, length);
117  }
118
119  virtual void SwapFrame(I420VideoFrame* video_frame) OVERRIDE {
120    I420VideoFrame* copy = NULL;
121    {
122      CriticalSectionScoped lock(crit_.get());
123      if (frame_pool_.size() > 0) {
124        copy = frame_pool_.front();
125        frame_pool_.pop_front();
126      }
127    }
128    if (copy == NULL)
129      copy = new I420VideoFrame();
130
131    copy->CopyFrame(*video_frame);
132    copy->set_timestamp(copy->render_time_ms() * 90);
133
134    {
135      CriticalSectionScoped lock(crit_.get());
136      if (first_send_frame_ == NULL && rtp_timestamp_delta_ == 0)
137        first_send_frame_ = copy;
138
139      frames_.push_back(copy);
140    }
141
142    input_->SwapFrame(video_frame);
143  }
144
145  virtual bool SendRtp(const uint8_t* packet, size_t length) OVERRIDE {
146    scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
147    RTPHeader header;
148    parser->Parse(packet, length, &header);
149
150    {
151      CriticalSectionScoped lock(crit_.get());
152      if (rtp_timestamp_delta_ == 0) {
153        rtp_timestamp_delta_ =
154            header.timestamp - first_send_frame_->timestamp();
155        first_send_frame_ = NULL;
156      }
157      send_times_[header.timestamp - rtp_timestamp_delta_] =
158          Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
159    }
160
161    return transport_->SendRtp(packet, length);
162  }
163
164  virtual bool SendRtcp(const uint8_t* packet, size_t length) OVERRIDE {
165    return transport_->SendRtcp(packet, length);
166  }
167
168  virtual void RenderFrame(const I420VideoFrame& video_frame,
169                           int time_to_render_ms) OVERRIDE {
170    int64_t render_time_ms =
171        Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
172    uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
173
174    CriticalSectionScoped lock(crit_.get());
175    while (frames_.front()->timestamp() < send_timestamp) {
176      AddFrameComparison(
177          frames_.front(), &last_rendered_frame_, true, render_time_ms);
178      frame_pool_.push_back(frames_.front());
179      frames_.pop_front();
180    }
181
182    I420VideoFrame* reference_frame = frames_.front();
183    frames_.pop_front();
184    assert(reference_frame != NULL);
185    EXPECT_EQ(reference_frame->timestamp(), send_timestamp);
186    assert(reference_frame->timestamp() == send_timestamp);
187
188    AddFrameComparison(reference_frame, &video_frame, false, render_time_ms);
189    frame_pool_.push_back(reference_frame);
190
191    last_rendered_frame_.CopyFrame(video_frame);
192  }
193
194  void Wait() {
195    EXPECT_EQ(kEventSignaled, done_->Wait(FullStackTest::kLongTimeoutMs));
196  }
197
198  VideoSendStreamInput* input_;
199  Transport* transport_;
200  PacketReceiver* receiver_;
201
202 private:
203  struct FrameComparison {
204    FrameComparison(const I420VideoFrame* reference,
205                    const I420VideoFrame* render,
206                    bool dropped,
207                    int64_t send_time_ms,
208                    int64_t recv_time_ms,
209                    int64_t render_time_ms)
210        : dropped(dropped),
211          send_time_ms(send_time_ms),
212          recv_time_ms(recv_time_ms),
213          render_time_ms(render_time_ms) {
214      this->reference.CopyFrame(*reference);
215      this->render.CopyFrame(*render);
216    }
217
218    FrameComparison(const FrameComparison& compare)
219        : dropped(compare.dropped),
220          send_time_ms(compare.send_time_ms),
221          recv_time_ms(compare.recv_time_ms),
222          render_time_ms(compare.render_time_ms) {
223      this->reference.CopyFrame(compare.reference);
224      this->render.CopyFrame(compare.render);
225    }
226
227    ~FrameComparison() {}
228
229    I420VideoFrame reference;
230    I420VideoFrame render;
231    bool dropped;
232    int64_t send_time_ms;
233    int64_t recv_time_ms;
234    int64_t render_time_ms;
235  };
236
237  void AddFrameComparison(const I420VideoFrame* reference,
238                          const I420VideoFrame* render,
239                          bool dropped,
240                          int64_t render_time_ms)
241      EXCLUSIVE_LOCKS_REQUIRED(crit_) {
242    int64_t send_time_ms = send_times_[reference->timestamp()];
243    send_times_.erase(reference->timestamp());
244    int64_t recv_time_ms = recv_times_[reference->timestamp()];
245    recv_times_.erase(reference->timestamp());
246
247    CriticalSectionScoped crit(comparison_lock_.get());
248    comparisons_.push_back(FrameComparison(reference,
249                                           render,
250                                           dropped,
251                                           send_time_ms,
252                                           recv_time_ms,
253                                           render_time_ms));
254  }
255
256  static bool FrameComparisonThread(void* obj) {
257    return static_cast<VideoAnalyzer*>(obj)->CompareFrames();
258  }
259
260  bool CompareFrames() {
261    assert(frames_left_ > 0);
262
263    I420VideoFrame reference;
264    I420VideoFrame render;
265    bool dropped;
266    int64_t send_time_ms;
267    int64_t recv_time_ms;
268    int64_t render_time_ms;
269
270    SleepMs(10);
271
272    while (true) {
273      {
274        CriticalSectionScoped crit(comparison_lock_.get());
275        if (comparisons_.empty())
276          return true;
277        reference.SwapFrame(&comparisons_.front().reference);
278        render.SwapFrame(&comparisons_.front().render);
279        dropped = comparisons_.front().dropped;
280        send_time_ms = comparisons_.front().send_time_ms;
281        recv_time_ms = comparisons_.front().recv_time_ms;
282        render_time_ms = comparisons_.front().render_time_ms;
283        comparisons_.pop_front();
284      }
285
286      PerformFrameComparison(&reference,
287                             &render,
288                             dropped,
289                             send_time_ms,
290                             recv_time_ms,
291                             render_time_ms);
292
293      if (--frames_left_ == 0) {
294        PrintResult("psnr", psnr_, " dB");
295        PrintResult("ssim", ssim_, "");
296        PrintResult("sender_time", sender_time_, " ms");
297        printf(
298            "RESULT dropped_frames: %s = %d\n", test_label_, dropped_frames_);
299        PrintResult("receiver_time", receiver_time_, " ms");
300        PrintResult("total_delay_incl_network", end_to_end_, " ms");
301        PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
302        EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
303        EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
304        done_->Set();
305
306        return false;
307      }
308    }
309  }
310
311  void PerformFrameComparison(const I420VideoFrame* reference,
312                              const I420VideoFrame* render,
313                              bool dropped,
314                              int64_t send_time_ms,
315                              int64_t recv_time_ms,
316                              int64_t render_time_ms) {
317    psnr_.AddSample(I420PSNR(reference, render));
318    ssim_.AddSample(I420SSIM(reference, render));
319    if (dropped) {
320      ++dropped_frames_;
321      return;
322    }
323    if (last_render_time_ != 0)
324      rendered_delta_.AddSample(render_time_ms - last_render_time_);
325    last_render_time_ = render_time_ms;
326
327    int64_t input_time_ms = reference->render_time_ms();
328    sender_time_.AddSample(send_time_ms - input_time_ms);
329    receiver_time_.AddSample(render_time_ms - recv_time_ms);
330    end_to_end_.AddSample(render_time_ms - input_time_ms);
331  }
332
333  void PrintResult(const char* result_type,
334                   test::Statistics stats,
335                   const char* unit) {
336    printf("RESULT %s: %s = {%f, %f}%s\n",
337           result_type,
338           test_label_,
339           stats.Mean(),
340           stats.StandardDeviation(),
341           unit);
342  }
343
344  const char* const test_label_;
345  test::Statistics sender_time_;
346  test::Statistics receiver_time_;
347  test::Statistics psnr_;
348  test::Statistics ssim_;
349  test::Statistics end_to_end_;
350  test::Statistics rendered_delta_;
351  int frames_left_;
352  int dropped_frames_;
353  int64_t last_render_time_;
354  uint32_t rtp_timestamp_delta_;
355
356  const scoped_ptr<CriticalSectionWrapper> crit_;
357  std::deque<I420VideoFrame*> frames_ GUARDED_BY(crit_);
358  std::deque<I420VideoFrame*> frame_pool_ GUARDED_BY(crit_);
359  I420VideoFrame last_rendered_frame_ GUARDED_BY(crit_);
360  std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);
361  std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);
362  I420VideoFrame* first_send_frame_ GUARDED_BY(crit_);
363  double avg_psnr_threshold_ GUARDED_BY(crit_);
364  double avg_ssim_threshold_ GUARDED_BY(crit_);
365
366  const scoped_ptr<CriticalSectionWrapper> comparison_lock_;
367  const scoped_ptr<ThreadWrapper> comparison_thread_;
368  std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
369  const scoped_ptr<EventWrapper> done_;
370};
371
372void FullStackTest::RunTest(const FullStackTestParams& params) {
373  test::DirectTransport send_transport(params.link);
374  test::DirectTransport recv_transport(params.link);
375  VideoAnalyzer analyzer(NULL,
376                         &send_transport,
377                         params.test_label,
378                         params.avg_psnr_threshold,
379                         params.avg_ssim_threshold,
380                         kFullStackTestDurationSecs * params.clip.fps);
381
382  CreateCalls(Call::Config(&analyzer), Call::Config(&recv_transport));
383
384  analyzer.SetReceiver(receiver_call_->Receiver());
385  send_transport.SetReceiver(&analyzer);
386  recv_transport.SetReceiver(sender_call_->Receiver());
387
388  CreateSendConfig(1);
389
390  scoped_ptr<VideoEncoder> encoder(
391      VideoEncoder::Create(VideoEncoder::kVp8));
392  send_config_.encoder_settings.encoder = encoder.get();
393  send_config_.encoder_settings.payload_name = "VP8";
394  send_config_.encoder_settings.payload_type = 124;
395
396  VideoStream* stream = &encoder_config_.streams[0];
397  stream->width = params.clip.width;
398  stream->height = params.clip.height;
399  stream->min_bitrate_bps = params.min_bitrate_bps;
400  stream->target_bitrate_bps = params.target_bitrate_bps;
401  stream->max_bitrate_bps = params.max_bitrate_bps;
402  stream->max_framerate = params.clip.fps;
403
404  CreateMatchingReceiveConfigs();
405  receive_configs_[0].renderer = &analyzer;
406
407  CreateStreams();
408  analyzer.input_ = send_stream_->Input();
409
410  frame_generator_capturer_.reset(
411      test::FrameGeneratorCapturer::CreateFromYuvFile(
412          &analyzer,
413          test::ResourcePath(params.clip.name, "yuv").c_str(),
414          params.clip.width,
415          params.clip.height,
416          params.clip.fps,
417          Clock::GetRealTimeClock()));
418
419  ASSERT_TRUE(frame_generator_capturer_.get() != NULL)
420      << "Could not create capturer for " << params.clip.name
421      << ".yuv. Is this resource file present?";
422
423  Start();
424
425  analyzer.Wait();
426
427  send_transport.StopSending();
428  recv_transport.StopSending();
429
430  Stop();
431
432  DestroyStreams();
433}
434
435TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) {
436  FullStackTestParams paris_qcif = {"net_delay_0_0_plr_0",
437                                    {"paris_qcif", 176, 144, 30},
438                                    300000,
439                                    300000,
440                                    300000,
441                                    36.0,
442                                    0.96
443                                   };
444  RunTest(paris_qcif);
445}
446
447TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) {
448  // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif.
449  FullStackTestParams foreman_cif = {"foreman_cif_net_delay_0_0_plr_0",
450                                     {"foreman_cif", 352, 288, 30},
451                                     700000,
452                                     700000,
453                                     700000,
454                                     0.0,
455                                     0.0
456                                    };
457  RunTest(foreman_cif);
458}
459
460TEST_F(FullStackTest, ForemanCif500kbps) {
461  FullStackTestParams foreman_cif = {"foreman_cif_500kbps",
462                                     {"foreman_cif", 352, 288, 30},
463                                     30000,
464                                     500000,
465                                     2000000,
466                                     0.0,
467                                     0.0
468                                    };
469  foreman_cif.link.queue_length_packets = 0;
470  foreman_cif.link.queue_delay_ms = 0;
471  foreman_cif.link.link_capacity_kbps = 500;
472  RunTest(foreman_cif);
473}
474
475TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) {
476  FullStackTestParams foreman_cif = {"foreman_cif_500kbps_32pkts_queue",
477                                     {"foreman_cif", 352, 288, 30},
478                                     30000,
479                                     500000,
480                                     2000000,
481                                     0.0,
482                                     0.0
483                                    };
484  foreman_cif.link.queue_length_packets = 32;
485  foreman_cif.link.queue_delay_ms = 0;
486  foreman_cif.link.link_capacity_kbps = 500;
487  RunTest(foreman_cif);
488}
489
490TEST_F(FullStackTest, ForemanCif500kbps100ms) {
491  FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms",
492                                     {"foreman_cif", 352, 288, 30},
493                                     30000,
494                                     500000,
495                                     2000000,
496                                     0.0,
497                                     0.0
498                                    };
499  foreman_cif.link.queue_length_packets = 0;
500  foreman_cif.link.queue_delay_ms = 100;
501  foreman_cif.link.link_capacity_kbps = 500;
502  RunTest(foreman_cif);
503}
504
505TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) {
506  FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms_32pkts_queue",
507                                     {"foreman_cif", 352, 288, 30},
508                                     30000,
509                                     500000,
510                                     2000000,
511                                     0.0,
512                                     0.0
513                                    };
514  foreman_cif.link.queue_length_packets = 32;
515  foreman_cif.link.queue_delay_ms = 100;
516  foreman_cif.link.link_capacity_kbps = 500;
517  RunTest(foreman_cif);
518}
519
520TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) {
521  FullStackTestParams foreman_cif = {"foreman_cif_1000kbps_100ms_32pkts_queue",
522                                     {"foreman_cif", 352, 288, 30},
523                                     30000,
524                                     2000000,
525                                     2000000,
526                                     0.0,
527                                     0.0
528                                    };
529  foreman_cif.link.queue_length_packets = 32;
530  foreman_cif.link.queue_delay_ms = 100;
531  foreman_cif.link.link_capacity_kbps = 1000;
532  RunTest(foreman_cif);
533}
534}  // namespace webrtc
535