1/*
2 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include <vector>
12
13#include "testing/gtest/include/gtest/gtest.h"
14#include "webrtc/base/scoped_ptr.h"
15#include "webrtc/common.h"
16#include "webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h"
17#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
18#include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
19#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
20#include "webrtc/modules/video_coding/include/mock/mock_vcm_callbacks.h"
21#include "webrtc/modules/video_coding/include/video_coding.h"
22#include "webrtc/modules/video_coding/video_coding_impl.h"
23#include "webrtc/modules/video_coding/test/test_util.h"
24#include "webrtc/system_wrappers/include/clock.h"
25#include "webrtc/test/frame_generator.h"
26#include "webrtc/test/testsupport/fileutils.h"
27
28using ::testing::_;
29using ::testing::AllOf;
30using ::testing::ElementsAre;
31using ::testing::ElementsAreArray;
32using ::testing::Field;
33using ::testing::NiceMock;
34using ::testing::Pointee;
35using ::testing::Return;
36using ::testing::FloatEq;
37using std::vector;
38using webrtc::test::FrameGenerator;
39
40namespace webrtc {
41namespace vcm {
42namespace {
43enum { kMaxNumberOfTemporalLayers = 3 };
44
45struct Vp8StreamInfo {
46  float framerate_fps[kMaxNumberOfTemporalLayers];
47  int bitrate_kbps[kMaxNumberOfTemporalLayers];
48};
49
50MATCHER_P(MatchesVp8StreamInfo, expected, "") {
51  bool res = true;
52  for (int tl = 0; tl < kMaxNumberOfTemporalLayers; ++tl) {
53    if (fabs(expected.framerate_fps[tl] - arg.framerate_fps[tl]) > 0.5) {
54      *result_listener << " framerate_fps[" << tl
55                       << "] = " << arg.framerate_fps[tl] << " (expected "
56                       << expected.framerate_fps[tl] << ") ";
57      res = false;
58    }
59    if (abs(expected.bitrate_kbps[tl] - arg.bitrate_kbps[tl]) > 10) {
60      *result_listener << " bitrate_kbps[" << tl
61                       << "] = " << arg.bitrate_kbps[tl] << " (expected "
62                       << expected.bitrate_kbps[tl] << ") ";
63      res = false;
64    }
65  }
66  return res;
67}
68
69class EmptyFrameGenerator : public FrameGenerator {
70 public:
71  EmptyFrameGenerator(int width, int height) : width_(width), height_(height) {}
72  VideoFrame* NextFrame() override {
73    frame_.reset(new VideoFrame());
74    frame_->CreateEmptyFrame(width_, height_, width_, (width_ + 1) / 2,
75                             (width_ + 1) / 2);
76    return frame_.get();
77  }
78
79 private:
80  const int width_;
81  const int height_;
82  rtc::scoped_ptr<VideoFrame> frame_;
83};
84
85class PacketizationCallback : public VCMPacketizationCallback {
86 public:
87  explicit PacketizationCallback(Clock* clock)
88      : clock_(clock), start_time_ms_(clock_->TimeInMilliseconds()) {}
89
90  virtual ~PacketizationCallback() {}
91
92  int32_t SendData(uint8_t payload_type,
93                   const EncodedImage& encoded_image,
94                   const RTPFragmentationHeader& fragmentation_header,
95                   const RTPVideoHeader* rtp_video_header) override {
96    assert(rtp_video_header);
97    frame_data_.push_back(FrameData(encoded_image._length, *rtp_video_header));
98    return 0;
99  }
100
101  void Reset() {
102    frame_data_.clear();
103    start_time_ms_ = clock_->TimeInMilliseconds();
104  }
105
106  float FramerateFpsWithinTemporalLayer(int temporal_layer) {
107    return CountFramesWithinTemporalLayer(temporal_layer) *
108           (1000.0 / interval_ms());
109  }
110
111  float BitrateKbpsWithinTemporalLayer(int temporal_layer) {
112    return SumPayloadBytesWithinTemporalLayer(temporal_layer) * 8.0 /
113           interval_ms();
114  }
115
116  Vp8StreamInfo CalculateVp8StreamInfo() {
117    Vp8StreamInfo info;
118    for (int tl = 0; tl < 3; ++tl) {
119      info.framerate_fps[tl] = FramerateFpsWithinTemporalLayer(tl);
120      info.bitrate_kbps[tl] = BitrateKbpsWithinTemporalLayer(tl);
121    }
122    return info;
123  }
124
125 private:
126  struct FrameData {
127    FrameData() {}
128
129    FrameData(size_t payload_size, const RTPVideoHeader& rtp_video_header)
130        : payload_size(payload_size), rtp_video_header(rtp_video_header) {}
131
132    size_t payload_size;
133    RTPVideoHeader rtp_video_header;
134  };
135
136  int64_t interval_ms() {
137    int64_t diff = (clock_->TimeInMilliseconds() - start_time_ms_);
138    EXPECT_GT(diff, 0);
139    return diff;
140  }
141
142  int CountFramesWithinTemporalLayer(int temporal_layer) {
143    int frames = 0;
144    for (size_t i = 0; i < frame_data_.size(); ++i) {
145      EXPECT_EQ(kRtpVideoVp8, frame_data_[i].rtp_video_header.codec);
146      const uint8_t temporal_idx =
147          frame_data_[i].rtp_video_header.codecHeader.VP8.temporalIdx;
148      if (temporal_idx <= temporal_layer || temporal_idx == kNoTemporalIdx)
149        frames++;
150    }
151    return frames;
152  }
153
154  size_t SumPayloadBytesWithinTemporalLayer(int temporal_layer) {
155    size_t payload_size = 0;
156    for (size_t i = 0; i < frame_data_.size(); ++i) {
157      EXPECT_EQ(kRtpVideoVp8, frame_data_[i].rtp_video_header.codec);
158      const uint8_t temporal_idx =
159          frame_data_[i].rtp_video_header.codecHeader.VP8.temporalIdx;
160      if (temporal_idx <= temporal_layer || temporal_idx == kNoTemporalIdx)
161        payload_size += frame_data_[i].payload_size;
162    }
163    return payload_size;
164  }
165
166  Clock* clock_;
167  int64_t start_time_ms_;
168  vector<FrameData> frame_data_;
169};
170
171class TestVideoSender : public ::testing::Test {
172 protected:
173  // Note: simulated clock starts at 1 seconds, since parts of webrtc use 0 as
174  // a special case (e.g. frame rate in media optimization).
175  TestVideoSender() : clock_(1000), packetization_callback_(&clock_) {}
176
177  void SetUp() override {
178    sender_.reset(
179        new VideoSender(&clock_, &post_encode_callback_, nullptr, nullptr));
180    EXPECT_EQ(0, sender_->RegisterTransportCallback(&packetization_callback_));
181  }
182
183  void AddFrame() {
184    assert(generator_.get());
185    sender_->AddVideoFrame(*generator_->NextFrame(), NULL, NULL);
186  }
187
188  SimulatedClock clock_;
189  PacketizationCallback packetization_callback_;
190  MockEncodedImageCallback post_encode_callback_;
191  // Used by subclassing tests, need to outlive sender_.
192  rtc::scoped_ptr<VideoEncoder> encoder_;
193  rtc::scoped_ptr<VideoSender> sender_;
194  rtc::scoped_ptr<FrameGenerator> generator_;
195};
196
197class TestVideoSenderWithMockEncoder : public TestVideoSender {
198 protected:
199  static const int kDefaultWidth = 1280;
200  static const int kDefaultHeight = 720;
201  static const int kNumberOfStreams = 3;
202  static const int kNumberOfLayers = 3;
203  static const int kUnusedPayloadType = 10;
204
205  void SetUp() override {
206    TestVideoSender::SetUp();
207    sender_->RegisterExternalEncoder(&encoder_, kUnusedPayloadType, false);
208    VideoCodingModule::Codec(kVideoCodecVP8, &settings_);
209    settings_.numberOfSimulcastStreams = kNumberOfStreams;
210    ConfigureStream(kDefaultWidth / 4, kDefaultHeight / 4, 100,
211                    &settings_.simulcastStream[0]);
212    ConfigureStream(kDefaultWidth / 2, kDefaultHeight / 2, 500,
213                    &settings_.simulcastStream[1]);
214    ConfigureStream(kDefaultWidth, kDefaultHeight, 1200,
215                    &settings_.simulcastStream[2]);
216    settings_.plType = kUnusedPayloadType;  // Use the mocked encoder.
217    generator_.reset(
218        new EmptyFrameGenerator(settings_.width, settings_.height));
219    EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200));
220  }
221
222  void TearDown() override { sender_.reset(); }
223
224  void ExpectIntraRequest(int stream) {
225    if (stream == -1) {
226      // No intra request expected.
227      EXPECT_CALL(
228          encoder_,
229          Encode(_, _, Pointee(ElementsAre(kVideoFrameDelta, kVideoFrameDelta,
230                                           kVideoFrameDelta))))
231          .Times(1)
232          .WillRepeatedly(Return(0));
233      return;
234    }
235    assert(stream >= 0);
236    assert(stream < kNumberOfStreams);
237    std::vector<FrameType> frame_types(kNumberOfStreams, kVideoFrameDelta);
238    frame_types[stream] = kVideoFrameKey;
239    EXPECT_CALL(encoder_,
240                Encode(_, _, Pointee(ElementsAreArray(&frame_types[0],
241                                                      frame_types.size()))))
242        .Times(1)
243        .WillRepeatedly(Return(0));
244  }
245
246  static void ConfigureStream(int width,
247                              int height,
248                              int max_bitrate,
249                              SimulcastStream* stream) {
250    assert(stream);
251    stream->width = width;
252    stream->height = height;
253    stream->maxBitrate = max_bitrate;
254    stream->numberOfTemporalLayers = kNumberOfLayers;
255    stream->qpMax = 45;
256  }
257
258  VideoCodec settings_;
259  NiceMock<MockVideoEncoder> encoder_;
260};
261
262TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequests) {
263  EXPECT_EQ(0, sender_->IntraFrameRequest(0));
264  ExpectIntraRequest(0);
265  AddFrame();
266  ExpectIntraRequest(-1);
267  AddFrame();
268
269  EXPECT_EQ(0, sender_->IntraFrameRequest(1));
270  ExpectIntraRequest(1);
271  AddFrame();
272  ExpectIntraRequest(-1);
273  AddFrame();
274
275  EXPECT_EQ(0, sender_->IntraFrameRequest(2));
276  ExpectIntraRequest(2);
277  AddFrame();
278  ExpectIntraRequest(-1);
279  AddFrame();
280
281  EXPECT_EQ(-1, sender_->IntraFrameRequest(3));
282  ExpectIntraRequest(-1);
283  AddFrame();
284
285  EXPECT_EQ(-1, sender_->IntraFrameRequest(-1));
286  ExpectIntraRequest(-1);
287  AddFrame();
288}
289
290TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequestsInternalCapture) {
291  // De-register current external encoder.
292  sender_->RegisterExternalEncoder(nullptr, kUnusedPayloadType, false);
293  // Register encoder with internal capture.
294  sender_->RegisterExternalEncoder(&encoder_, kUnusedPayloadType, true);
295  EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200));
296  ExpectIntraRequest(0);
297  EXPECT_EQ(0, sender_->IntraFrameRequest(0));
298  ExpectIntraRequest(1);
299  EXPECT_EQ(0, sender_->IntraFrameRequest(1));
300  ExpectIntraRequest(2);
301  EXPECT_EQ(0, sender_->IntraFrameRequest(2));
302  // No requests expected since these indices are out of bounds.
303  EXPECT_EQ(-1, sender_->IntraFrameRequest(3));
304  EXPECT_EQ(-1, sender_->IntraFrameRequest(-1));
305}
306
307TEST_F(TestVideoSenderWithMockEncoder, EncoderFramerateUpdatedViaProcess) {
308  sender_->SetChannelParameters(settings_.startBitrate * 1000, 0, 200);
309  const int64_t kRateStatsWindowMs = 2000;
310  const uint32_t kInputFps = 20;
311  int64_t start_time = clock_.TimeInMilliseconds();
312  while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) {
313    AddFrame();
314    clock_.AdvanceTimeMilliseconds(1000 / kInputFps);
315  }
316  EXPECT_CALL(encoder_, SetRates(_, kInputFps)).Times(1).WillOnce(Return(0));
317  sender_->Process();
318  AddFrame();
319}
320
321TEST_F(TestVideoSenderWithMockEncoder,
322       NoRedundantSetChannelParameterOrSetRatesCalls) {
323  const uint8_t kLossRate = 4;
324  const uint8_t kRtt = 200;
325  const int64_t kRateStatsWindowMs = 2000;
326  const uint32_t kInputFps = 20;
327  int64_t start_time = clock_.TimeInMilliseconds();
328  // Expect initial call to SetChannelParameters. Rates are initialized through
329  // InitEncode and expects no additional call before the framerate (or bitrate)
330  // updates.
331  EXPECT_CALL(encoder_, SetChannelParameters(kLossRate, kRtt))
332      .Times(1)
333      .WillOnce(Return(0));
334  sender_->SetChannelParameters(settings_.startBitrate * 1000, kLossRate, kRtt);
335  while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) {
336    AddFrame();
337    clock_.AdvanceTimeMilliseconds(1000 / kInputFps);
338  }
339  // After process, input framerate should be updated but not ChannelParameters
340  // as they are the same as before.
341  EXPECT_CALL(encoder_, SetRates(_, kInputFps)).Times(1).WillOnce(Return(0));
342  sender_->Process();
343  AddFrame();
344  // Call to SetChannelParameters with changed bitrate should call encoder
345  // SetRates but not encoder SetChannelParameters (that are unchanged).
346  EXPECT_CALL(encoder_, SetRates(2 * settings_.startBitrate, kInputFps))
347      .Times(1)
348      .WillOnce(Return(0));
349  sender_->SetChannelParameters(2 * settings_.startBitrate * 1000, kLossRate,
350                                kRtt);
351  AddFrame();
352}
353
354class TestVideoSenderWithVp8 : public TestVideoSender {
355 public:
356  TestVideoSenderWithVp8()
357      : codec_bitrate_kbps_(300), available_bitrate_kbps_(1000) {}
358
359  void SetUp() override {
360    TestVideoSender::SetUp();
361
362    const char* input_video = "foreman_cif";
363    const int width = 352;
364    const int height = 288;
365    generator_.reset(FrameGenerator::CreateFromYuvFile(
366        std::vector<std::string>(1, test::ResourcePath(input_video, "yuv")),
367        width, height, 1));
368
369    codec_ = MakeVp8VideoCodec(width, height, 3);
370    codec_.minBitrate = 10;
371    codec_.startBitrate = codec_bitrate_kbps_;
372    codec_.maxBitrate = codec_bitrate_kbps_;
373    encoder_.reset(VP8Encoder::Create());
374    sender_->RegisterExternalEncoder(encoder_.get(), codec_.plType, false);
375    EXPECT_EQ(0, sender_->RegisterSendCodec(&codec_, 1, 1200));
376  }
377
378  static VideoCodec MakeVp8VideoCodec(int width,
379                                      int height,
380                                      int temporal_layers) {
381    VideoCodec codec;
382    VideoCodingModule::Codec(kVideoCodecVP8, &codec);
383    codec.width = width;
384    codec.height = height;
385    codec.codecSpecific.VP8.numberOfTemporalLayers = temporal_layers;
386    return codec;
387  }
388
389  void InsertFrames(float framerate, float seconds) {
390    for (int i = 0; i < seconds * framerate; ++i) {
391      clock_.AdvanceTimeMilliseconds(1000.0f / framerate);
392      EXPECT_CALL(post_encode_callback_, Encoded(_, NULL, NULL))
393          .WillOnce(Return(0));
394      AddFrame();
395      // SetChannelParameters needs to be called frequently to propagate
396      // framerate from the media optimization into the encoder.
397      // Note: SetChannelParameters fails if less than 2 frames are in the
398      // buffer since it will fail to calculate the framerate.
399      if (i != 0) {
400        EXPECT_EQ(VCM_OK, sender_->SetChannelParameters(
401                              available_bitrate_kbps_ * 1000, 0, 200));
402      }
403    }
404  }
405
406  Vp8StreamInfo SimulateWithFramerate(float framerate) {
407    const float short_simulation_interval = 5.0;
408    const float long_simulation_interval = 10.0;
409    // It appears that this 5 seconds simulation is needed to allow
410    // bitrate and framerate to stabilize.
411    InsertFrames(framerate, short_simulation_interval);
412    packetization_callback_.Reset();
413
414    InsertFrames(framerate, long_simulation_interval);
415    return packetization_callback_.CalculateVp8StreamInfo();
416  }
417
418 protected:
419  VideoCodec codec_;
420  int codec_bitrate_kbps_;
421  int available_bitrate_kbps_;
422};
423
424#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
425#define MAYBE_FixedTemporalLayersStrategy DISABLED_FixedTemporalLayersStrategy
426#else
427#define MAYBE_FixedTemporalLayersStrategy FixedTemporalLayersStrategy
428#endif
429TEST_F(TestVideoSenderWithVp8, MAYBE_FixedTemporalLayersStrategy) {
430  const int low_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0];
431  const int mid_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1];
432  const int high_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2];
433  {
434    Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
435    EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));
436  }
437  {
438    Vp8StreamInfo expected = {{3.75, 7.5, 15.0}, {low_b, mid_b, high_b}};
439    EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected));
440  }
441}
442
443#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
444#define MAYBE_RealTimeTemporalLayersStrategy \
445  DISABLED_RealTimeTemporalLayersStrategy
446#else
447#define MAYBE_RealTimeTemporalLayersStrategy RealTimeTemporalLayersStrategy
448#endif
449TEST_F(TestVideoSenderWithVp8, MAYBE_RealTimeTemporalLayersStrategy) {
450  Config extra_options;
451  extra_options.Set<TemporalLayers::Factory>(
452      new RealTimeTemporalLayersFactory());
453  VideoCodec codec = MakeVp8VideoCodec(352, 288, 3);
454  codec.extra_options = &extra_options;
455  codec.minBitrate = 10;
456  codec.startBitrate = codec_bitrate_kbps_;
457  codec.maxBitrate = codec_bitrate_kbps_;
458  EXPECT_EQ(0, sender_->RegisterSendCodec(&codec, 1, 1200));
459
460  const int low_b = codec_bitrate_kbps_ * 0.4;
461  const int mid_b = codec_bitrate_kbps_ * 0.6;
462  const int high_b = codec_bitrate_kbps_;
463
464  {
465    Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
466    EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));
467  }
468  {
469    Vp8StreamInfo expected = {{5.0, 10.0, 20.0}, {low_b, mid_b, high_b}};
470    EXPECT_THAT(SimulateWithFramerate(20.0), MatchesVp8StreamInfo(expected));
471  }
472  {
473    Vp8StreamInfo expected = {{7.5, 15.0, 15.0}, {mid_b, high_b, high_b}};
474    EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected));
475  }
476  {
477    Vp8StreamInfo expected = {{5.0, 10.0, 10.0}, {mid_b, high_b, high_b}};
478    EXPECT_THAT(SimulateWithFramerate(10.0), MatchesVp8StreamInfo(expected));
479  }
480  {
481    // TODO(andresp): Find out why this fails with framerate = 7.5
482    Vp8StreamInfo expected = {{7.0, 7.0, 7.0}, {high_b, high_b, high_b}};
483    EXPECT_THAT(SimulateWithFramerate(7.0), MatchesVp8StreamInfo(expected));
484  }
485}
486}  // namespace
487}  // namespace vcm
488}  // namespace webrtc
489