1/*
2 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/call/rampup_tests.h"
12
13#include "testing/gtest/include/gtest/gtest.h"
14#include "webrtc/base/checks.h"
15#include "webrtc/base/platform_thread.h"
16#include "webrtc/test/testsupport/perf_test.h"
17
18namespace webrtc {
19namespace {
20
21static const int64_t kPollIntervalMs = 20;
22
23std::vector<uint32_t> GenerateSsrcs(size_t num_streams, uint32_t ssrc_offset) {
24  std::vector<uint32_t> ssrcs;
25  for (size_t i = 0; i != num_streams; ++i)
26    ssrcs.push_back(static_cast<uint32_t>(ssrc_offset + i));
27  return ssrcs;
28}
29}  // namespace
30
31RampUpTester::RampUpTester(size_t num_video_streams,
32                           size_t num_audio_streams,
33                           unsigned int start_bitrate_bps,
34                           const std::string& extension_type,
35                           bool rtx,
36                           bool red)
37    : EndToEndTest(test::CallTest::kLongTimeoutMs),
38      event_(false, false),
39      clock_(Clock::GetRealTimeClock()),
40      num_video_streams_(num_video_streams),
41      num_audio_streams_(num_audio_streams),
42      rtx_(rtx),
43      red_(red),
44      send_stream_(nullptr),
45      start_bitrate_bps_(start_bitrate_bps),
46      start_bitrate_verified_(false),
47      expected_bitrate_bps_(0),
48      test_start_ms_(-1),
49      ramp_up_finished_ms_(-1),
50      extension_type_(extension_type),
51      video_ssrcs_(GenerateSsrcs(num_video_streams_, 100)),
52      video_rtx_ssrcs_(GenerateSsrcs(num_video_streams_, 200)),
53      audio_ssrcs_(GenerateSsrcs(num_audio_streams_, 300)),
54      poller_thread_(&BitrateStatsPollingThread,
55                     this,
56                     "BitrateStatsPollingThread"),
57      sender_call_(nullptr) {
58  EXPECT_LE(num_audio_streams_, 1u);
59  if (rtx_) {
60    for (size_t i = 0; i < video_ssrcs_.size(); ++i)
61      rtx_ssrc_map_[video_rtx_ssrcs_[i]] = video_ssrcs_[i];
62  }
63}
64
65RampUpTester::~RampUpTester() {
66  event_.Set();
67}
68
69Call::Config RampUpTester::GetSenderCallConfig() {
70  Call::Config call_config;
71  if (start_bitrate_bps_ != 0) {
72    call_config.bitrate_config.start_bitrate_bps = start_bitrate_bps_;
73  }
74  call_config.bitrate_config.min_bitrate_bps = 10000;
75  return call_config;
76}
77
78void RampUpTester::OnVideoStreamsCreated(
79    VideoSendStream* send_stream,
80    const std::vector<VideoReceiveStream*>& receive_streams) {
81  send_stream_ = send_stream;
82}
83
84test::PacketTransport* RampUpTester::CreateSendTransport(Call* sender_call) {
85  send_transport_ = new test::PacketTransport(sender_call, this,
86                                              test::PacketTransport::kSender,
87                                              forward_transport_config_);
88  return send_transport_;
89}
90
91size_t RampUpTester::GetNumVideoStreams() const {
92  return num_video_streams_;
93}
94
95size_t RampUpTester::GetNumAudioStreams() const {
96  return num_audio_streams_;
97}
98
99void RampUpTester::ModifyVideoConfigs(
100    VideoSendStream::Config* send_config,
101    std::vector<VideoReceiveStream::Config>* receive_configs,
102    VideoEncoderConfig* encoder_config) {
103  send_config->suspend_below_min_bitrate = true;
104
105  if (num_video_streams_ == 1) {
106    encoder_config->streams[0].target_bitrate_bps =
107        encoder_config->streams[0].max_bitrate_bps = 2000000;
108    // For single stream rampup until 1mbps
109    expected_bitrate_bps_ = kSingleStreamTargetBps;
110  } else {
111    // For multi stream rampup until all streams are being sent. That means
112    // enough birate to send all the target streams plus the min bitrate of
113    // the last one.
114    expected_bitrate_bps_ = encoder_config->streams.back().min_bitrate_bps;
115    for (size_t i = 0; i < encoder_config->streams.size() - 1; ++i) {
116      expected_bitrate_bps_ += encoder_config->streams[i].target_bitrate_bps;
117    }
118  }
119
120  send_config->rtp.extensions.clear();
121
122  bool remb;
123  bool transport_cc;
124  if (extension_type_ == RtpExtension::kAbsSendTime) {
125    remb = true;
126    transport_cc = false;
127    send_config->rtp.extensions.push_back(
128        RtpExtension(extension_type_.c_str(), kAbsSendTimeExtensionId));
129  } else if (extension_type_ == RtpExtension::kTransportSequenceNumber) {
130    remb = false;
131    transport_cc = true;
132    send_config->rtp.extensions.push_back(RtpExtension(
133        extension_type_.c_str(), kTransportSequenceNumberExtensionId));
134  } else {
135    remb = true;
136    transport_cc = false;
137    send_config->rtp.extensions.push_back(RtpExtension(
138        extension_type_.c_str(), kTransmissionTimeOffsetExtensionId));
139  }
140
141  send_config->rtp.nack.rtp_history_ms = test::CallTest::kNackRtpHistoryMs;
142  send_config->rtp.ssrcs = video_ssrcs_;
143  if (rtx_) {
144    send_config->rtp.rtx.payload_type = test::CallTest::kSendRtxPayloadType;
145    send_config->rtp.rtx.ssrcs = video_rtx_ssrcs_;
146  }
147  if (red_) {
148    send_config->rtp.fec.ulpfec_payload_type =
149        test::CallTest::kUlpfecPayloadType;
150    send_config->rtp.fec.red_payload_type = test::CallTest::kRedPayloadType;
151  }
152
153  size_t i = 0;
154  for (VideoReceiveStream::Config& recv_config : *receive_configs) {
155    recv_config.rtp.remb = remb;
156    recv_config.rtp.transport_cc = transport_cc;
157    recv_config.rtp.extensions = send_config->rtp.extensions;
158
159    recv_config.rtp.remote_ssrc = video_ssrcs_[i];
160    recv_config.rtp.nack.rtp_history_ms = send_config->rtp.nack.rtp_history_ms;
161
162    if (red_) {
163      recv_config.rtp.fec.red_payload_type =
164          send_config->rtp.fec.red_payload_type;
165      recv_config.rtp.fec.ulpfec_payload_type =
166          send_config->rtp.fec.ulpfec_payload_type;
167    }
168
169    if (rtx_) {
170      recv_config.rtp.rtx[send_config->encoder_settings.payload_type].ssrc =
171          video_rtx_ssrcs_[i];
172      recv_config.rtp.rtx[send_config->encoder_settings.payload_type]
173          .payload_type = send_config->rtp.rtx.payload_type;
174    }
175    ++i;
176  }
177}
178
179void RampUpTester::ModifyAudioConfigs(
180    AudioSendStream::Config* send_config,
181    std::vector<AudioReceiveStream::Config>* receive_configs) {
182  if (num_audio_streams_ == 0)
183    return;
184
185  EXPECT_NE(RtpExtension::kTOffset, extension_type_)
186      << "Audio BWE not supported with toffset.";
187
188  send_config->rtp.ssrc = audio_ssrcs_[0];
189  send_config->rtp.extensions.clear();
190
191  bool transport_cc = false;
192  if (extension_type_ == RtpExtension::kAbsSendTime) {
193    transport_cc = false;
194    send_config->rtp.extensions.push_back(
195        RtpExtension(extension_type_.c_str(), kAbsSendTimeExtensionId));
196  } else if (extension_type_ == RtpExtension::kTransportSequenceNumber) {
197    transport_cc = true;
198    send_config->rtp.extensions.push_back(RtpExtension(
199        extension_type_.c_str(), kTransportSequenceNumberExtensionId));
200  }
201
202  for (AudioReceiveStream::Config& recv_config : *receive_configs) {
203    recv_config.combined_audio_video_bwe = true;
204    recv_config.rtp.transport_cc = transport_cc;
205    recv_config.rtp.extensions = send_config->rtp.extensions;
206    recv_config.rtp.remote_ssrc = send_config->rtp.ssrc;
207  }
208}
209
210void RampUpTester::OnCallsCreated(Call* sender_call, Call* receiver_call) {
211  sender_call_ = sender_call;
212}
213
214bool RampUpTester::BitrateStatsPollingThread(void* obj) {
215  return static_cast<RampUpTester*>(obj)->PollStats();
216}
217
218bool RampUpTester::PollStats() {
219  if (sender_call_) {
220    Call::Stats stats = sender_call_->GetStats();
221
222    RTC_DCHECK_GT(expected_bitrate_bps_, 0);
223    if (!start_bitrate_verified_ && start_bitrate_bps_ != 0) {
224      // For tests with an explicitly set start bitrate, verify the first
225      // bitrate estimate is close to the start bitrate and lower than the
226      // test target bitrate. This is to verify a call respects the configured
227      // start bitrate, but due to the BWE implementation we can't guarantee the
228      // first estimate really is as high as the start bitrate.
229      EXPECT_GT(stats.send_bandwidth_bps, 0.9 * start_bitrate_bps_);
230      start_bitrate_verified_ = true;
231    }
232    if (stats.send_bandwidth_bps >= expected_bitrate_bps_) {
233      ramp_up_finished_ms_ = clock_->TimeInMilliseconds();
234      observation_complete_.Set();
235    }
236  }
237
238  return !event_.Wait(kPollIntervalMs);
239}
240
241void RampUpTester::ReportResult(const std::string& measurement,
242                                size_t value,
243                                const std::string& units) const {
244  webrtc::test::PrintResult(
245      measurement, "",
246      ::testing::UnitTest::GetInstance()->current_test_info()->name(), value,
247      units, false);
248}
249
250void RampUpTester::AccumulateStats(const VideoSendStream::StreamStats& stream,
251                                   size_t* total_packets_sent,
252                                   size_t* total_sent,
253                                   size_t* padding_sent,
254                                   size_t* media_sent) const {
255  *total_packets_sent += stream.rtp_stats.transmitted.packets +
256                         stream.rtp_stats.retransmitted.packets +
257                         stream.rtp_stats.fec.packets;
258  *total_sent += stream.rtp_stats.transmitted.TotalBytes() +
259                 stream.rtp_stats.retransmitted.TotalBytes() +
260                 stream.rtp_stats.fec.TotalBytes();
261  *padding_sent += stream.rtp_stats.transmitted.padding_bytes +
262                   stream.rtp_stats.retransmitted.padding_bytes +
263                   stream.rtp_stats.fec.padding_bytes;
264  *media_sent += stream.rtp_stats.MediaPayloadBytes();
265}
266
267void RampUpTester::TriggerTestDone() {
268  RTC_DCHECK_GE(test_start_ms_, 0);
269
270  // TODO(holmer): Add audio send stats here too when those APIs are available.
271  VideoSendStream::Stats send_stats = send_stream_->GetStats();
272
273  size_t total_packets_sent = 0;
274  size_t total_sent = 0;
275  size_t padding_sent = 0;
276  size_t media_sent = 0;
277  for (uint32_t ssrc : video_ssrcs_) {
278    AccumulateStats(send_stats.substreams[ssrc], &total_packets_sent,
279                    &total_sent, &padding_sent, &media_sent);
280  }
281
282  size_t rtx_total_packets_sent = 0;
283  size_t rtx_total_sent = 0;
284  size_t rtx_padding_sent = 0;
285  size_t rtx_media_sent = 0;
286  for (uint32_t rtx_ssrc : video_rtx_ssrcs_) {
287    AccumulateStats(send_stats.substreams[rtx_ssrc], &rtx_total_packets_sent,
288                    &rtx_total_sent, &rtx_padding_sent, &rtx_media_sent);
289  }
290
291  ReportResult("ramp-up-total-packets-sent", total_packets_sent, "packets");
292  ReportResult("ramp-up-total-sent", total_sent, "bytes");
293  ReportResult("ramp-up-media-sent", media_sent, "bytes");
294  ReportResult("ramp-up-padding-sent", padding_sent, "bytes");
295  ReportResult("ramp-up-rtx-total-packets-sent", rtx_total_packets_sent,
296               "packets");
297  ReportResult("ramp-up-rtx-total-sent", rtx_total_sent, "bytes");
298  ReportResult("ramp-up-rtx-media-sent", rtx_media_sent, "bytes");
299  ReportResult("ramp-up-rtx-padding-sent", rtx_padding_sent, "bytes");
300  if (ramp_up_finished_ms_ >= 0) {
301    ReportResult("ramp-up-time", ramp_up_finished_ms_ - test_start_ms_,
302                 "milliseconds");
303  }
304  ReportResult("ramp-up-average-network-latency",
305               send_transport_->GetAverageDelayMs(), "milliseconds");
306}
307
308void RampUpTester::PerformTest() {
309  test_start_ms_ = clock_->TimeInMilliseconds();
310  poller_thread_.Start();
311  EXPECT_TRUE(Wait()) << "Timed out while waiting for ramp-up to complete.";
312  TriggerTestDone();
313  poller_thread_.Stop();
314}
315
316RampUpDownUpTester::RampUpDownUpTester(size_t num_video_streams,
317                                       size_t num_audio_streams,
318                                       unsigned int start_bitrate_bps,
319                                       const std::string& extension_type,
320                                       bool rtx,
321                                       bool red)
322    : RampUpTester(num_video_streams,
323                   num_audio_streams,
324                   start_bitrate_bps,
325                   extension_type,
326                   rtx,
327                   red),
328      test_state_(kFirstRampup),
329      state_start_ms_(clock_->TimeInMilliseconds()),
330      interval_start_ms_(clock_->TimeInMilliseconds()),
331      sent_bytes_(0) {
332  forward_transport_config_.link_capacity_kbps = kHighBandwidthLimitBps / 1000;
333}
334
335RampUpDownUpTester::~RampUpDownUpTester() {}
336
337bool RampUpDownUpTester::PollStats() {
338  if (send_stream_) {
339    webrtc::VideoSendStream::Stats stats = send_stream_->GetStats();
340    int transmit_bitrate_bps = 0;
341    for (auto it : stats.substreams) {
342      transmit_bitrate_bps += it.second.total_bitrate_bps;
343    }
344
345    EvolveTestState(transmit_bitrate_bps, stats.suspended);
346  }
347
348  return !event_.Wait(kPollIntervalMs);
349}
350
351Call::Config RampUpDownUpTester::GetReceiverCallConfig() {
352  Call::Config config;
353  config.bitrate_config.min_bitrate_bps = 10000;
354  return config;
355}
356
357std::string RampUpDownUpTester::GetModifierString() const {
358  std::string str("_");
359  if (num_video_streams_ > 0) {
360    std::ostringstream s;
361    s << num_video_streams_;
362    str += s.str();
363    str += "stream";
364    str += (num_video_streams_ > 1 ? "s" : "");
365    str += "_";
366  }
367  if (num_audio_streams_ > 0) {
368    std::ostringstream s;
369    s << num_audio_streams_;
370    str += s.str();
371    str += "stream";
372    str += (num_audio_streams_ > 1 ? "s" : "");
373    str += "_";
374  }
375  str += (rtx_ ? "" : "no");
376  str += "rtx";
377  return str;
378}
379
380void RampUpDownUpTester::EvolveTestState(int bitrate_bps, bool suspended) {
381  int64_t now = clock_->TimeInMilliseconds();
382  switch (test_state_) {
383    case kFirstRampup: {
384      EXPECT_FALSE(suspended);
385      if (bitrate_bps > kExpectedHighBitrateBps) {
386        // The first ramp-up has reached the target bitrate. Change the
387        // channel limit, and move to the next test state.
388        forward_transport_config_.link_capacity_kbps =
389            kLowBandwidthLimitBps / 1000;
390        send_transport_->SetConfig(forward_transport_config_);
391        test_state_ = kLowRate;
392        webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
393                                  "first_rampup", now - state_start_ms_, "ms",
394                                  false);
395        state_start_ms_ = now;
396        interval_start_ms_ = now;
397        sent_bytes_ = 0;
398      }
399      break;
400    }
401    case kLowRate: {
402      if (bitrate_bps < kExpectedLowBitrateBps && suspended) {
403        // The ramp-down was successful. Change the channel limit back to a
404        // high value, and move to the next test state.
405        forward_transport_config_.link_capacity_kbps =
406            kHighBandwidthLimitBps / 1000;
407        send_transport_->SetConfig(forward_transport_config_);
408        test_state_ = kSecondRampup;
409        webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
410                                  "rampdown", now - state_start_ms_, "ms",
411                                  false);
412        state_start_ms_ = now;
413        interval_start_ms_ = now;
414        sent_bytes_ = 0;
415      }
416      break;
417    }
418    case kSecondRampup: {
419      if (bitrate_bps > kExpectedHighBitrateBps && !suspended) {
420        webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
421                                  "second_rampup", now - state_start_ms_, "ms",
422                                  false);
423        ReportResult("ramp-up-down-up-average-network-latency",
424                     send_transport_->GetAverageDelayMs(), "milliseconds");
425        observation_complete_.Set();
426      }
427      break;
428    }
429  }
430}
431
432class RampUpTest : public test::CallTest {
433 public:
434  RampUpTest() {}
435
436  virtual ~RampUpTest() {
437    EXPECT_EQ(nullptr, video_send_stream_);
438    EXPECT_TRUE(video_receive_streams_.empty());
439  }
440};
441
442TEST_F(RampUpTest, SingleStream) {
443  RampUpTester test(1, 0, 0, RtpExtension::kTOffset, false, false);
444  RunBaseTest(&test);
445}
446
447TEST_F(RampUpTest, Simulcast) {
448  RampUpTester test(3, 0, 0, RtpExtension::kTOffset, false, false);
449  RunBaseTest(&test);
450}
451
452TEST_F(RampUpTest, SimulcastWithRtx) {
453  RampUpTester test(3, 0, 0, RtpExtension::kTOffset, true, false);
454  RunBaseTest(&test);
455}
456
457TEST_F(RampUpTest, SimulcastByRedWithRtx) {
458  RampUpTester test(3, 0, 0, RtpExtension::kTOffset, true, true);
459  RunBaseTest(&test);
460}
461
462TEST_F(RampUpTest, SingleStreamWithHighStartBitrate) {
463  RampUpTester test(1, 0, 0.9 * kSingleStreamTargetBps, RtpExtension::kTOffset,
464                    false, false);
465  RunBaseTest(&test);
466}
467
468// Disabled on Mac due to flakiness, see
469// https://bugs.chromium.org/p/webrtc/issues/detail?id=5407
470#ifndef WEBRTC_MAC
471
472static const uint32_t kStartBitrateBps = 60000;
473
474TEST_F(RampUpTest, UpDownUpOneStream) {
475  RampUpDownUpTester test(1, 0, kStartBitrateBps, RtpExtension::kAbsSendTime,
476                          false, false);
477  RunBaseTest(&test);
478}
479
480TEST_F(RampUpTest, UpDownUpThreeStreams) {
481  RampUpDownUpTester test(3, 0, kStartBitrateBps, RtpExtension::kAbsSendTime,
482                          false, false);
483  RunBaseTest(&test);
484}
485
486TEST_F(RampUpTest, UpDownUpOneStreamRtx) {
487  RampUpDownUpTester test(1, 0, kStartBitrateBps, RtpExtension::kAbsSendTime,
488                          true, false);
489  RunBaseTest(&test);
490}
491
492TEST_F(RampUpTest, UpDownUpThreeStreamsRtx) {
493  RampUpDownUpTester test(3, 0, kStartBitrateBps, RtpExtension::kAbsSendTime,
494                          true, false);
495  RunBaseTest(&test);
496}
497
498TEST_F(RampUpTest, UpDownUpOneStreamByRedRtx) {
499  RampUpDownUpTester test(1, 0, kStartBitrateBps, RtpExtension::kAbsSendTime,
500                          true, true);
501  RunBaseTest(&test);
502}
503
504TEST_F(RampUpTest, UpDownUpThreeStreamsByRedRtx) {
505  RampUpDownUpTester test(3, 0, kStartBitrateBps, RtpExtension::kAbsSendTime,
506                          true, true);
507  RunBaseTest(&test);
508}
509
510TEST_F(RampUpTest, SendSideVideoUpDownUpRtx) {
511  RampUpDownUpTester test(3, 0, kStartBitrateBps,
512                          RtpExtension::kTransportSequenceNumber, true, false);
513  RunBaseTest(&test);
514}
515
516// TODO(holmer): Enable when audio bitrates are included in the bitrate
517//               allocation.
518TEST_F(RampUpTest, DISABLED_SendSideAudioVideoUpDownUpRtx) {
519  RampUpDownUpTester test(3, 1, kStartBitrateBps,
520                          RtpExtension::kTransportSequenceNumber, true, false);
521  RunBaseTest(&test);
522}
523
524#endif
525
526TEST_F(RampUpTest, AbsSendTimeSingleStream) {
527  RampUpTester test(1, 0, 0, RtpExtension::kAbsSendTime, false, false);
528  RunBaseTest(&test);
529}
530
531TEST_F(RampUpTest, AbsSendTimeSimulcast) {
532  RampUpTester test(3, 0, 0, RtpExtension::kAbsSendTime, false, false);
533  RunBaseTest(&test);
534}
535
536TEST_F(RampUpTest, AbsSendTimeSimulcastWithRtx) {
537  RampUpTester test(3, 0, 0, RtpExtension::kAbsSendTime, true, false);
538  RunBaseTest(&test);
539}
540
541TEST_F(RampUpTest, AbsSendTimeSimulcastByRedWithRtx) {
542  RampUpTester test(3, 0, 0, RtpExtension::kAbsSendTime, true, true);
543  RunBaseTest(&test);
544}
545
546TEST_F(RampUpTest, AbsSendTimeSingleStreamWithHighStartBitrate) {
547  RampUpTester test(1, 0, 0.9 * kSingleStreamTargetBps,
548                    RtpExtension::kAbsSendTime, false, false);
549  RunBaseTest(&test);
550}
551
552TEST_F(RampUpTest, TransportSequenceNumberSingleStream) {
553  RampUpTester test(1, 0, 0, RtpExtension::kTransportSequenceNumber, false,
554                    false);
555  RunBaseTest(&test);
556}
557
558TEST_F(RampUpTest, TransportSequenceNumberSimulcast) {
559  RampUpTester test(3, 0, 0, RtpExtension::kTransportSequenceNumber, false,
560                    false);
561  RunBaseTest(&test);
562}
563
564TEST_F(RampUpTest, TransportSequenceNumberSimulcastWithRtx) {
565  RampUpTester test(3, 0, 0, RtpExtension::kTransportSequenceNumber, true,
566                    false);
567  RunBaseTest(&test);
568}
569
570TEST_F(RampUpTest, AudioVideoTransportSequenceNumberSimulcastWithRtx) {
571  RampUpTester test(3, 1, 0, RtpExtension::kTransportSequenceNumber, true,
572                    false);
573  RunBaseTest(&test);
574}
575
576TEST_F(RampUpTest, TransportSequenceNumberSimulcastByRedWithRtx) {
577  RampUpTester test(3, 0, 0, RtpExtension::kTransportSequenceNumber, true,
578                    true);
579  RunBaseTest(&test);
580}
581
582TEST_F(RampUpTest, TransportSequenceNumberSingleStreamWithHighStartBitrate) {
583  RampUpTester test(1, 0, 0.9 * kSingleStreamTargetBps,
584                    RtpExtension::kTransportSequenceNumber, false, false);
585  RunBaseTest(&test);
586}
587}  // namespace webrtc
588