1/*
2 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/video/video_send_stream.h"
12
13#include <algorithm>
14#include <sstream>
15#include <string>
16#include <vector>
17
18#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
19#include "webrtc/system_wrappers/interface/logging.h"
20#include "webrtc/video_engine/include/vie_base.h"
21#include "webrtc/video_engine/include/vie_capture.h"
22#include "webrtc/video_engine/include/vie_codec.h"
23#include "webrtc/video_engine/include/vie_external_codec.h"
24#include "webrtc/video_engine/include/vie_image_process.h"
25#include "webrtc/video_engine/include/vie_network.h"
26#include "webrtc/video_engine/include/vie_rtp_rtcp.h"
27#include "webrtc/video_engine/vie_defines.h"
28#include "webrtc/video_send_stream.h"
29
30namespace webrtc {
31std::string
32VideoSendStream::Config::EncoderSettings::ToString() const {
33  std::stringstream ss;
34  ss << "{payload_name: " << payload_name;
35  ss << ", payload_type: " << payload_type;
36  if (encoder != NULL)
37    ss << ", encoder: " << (encoder != NULL ? "(encoder)" : "NULL");
38  ss << '}';
39  return ss.str();
40}
41
42std::string VideoSendStream::Config::Rtp::Rtx::ToString()
43    const {
44  std::stringstream ss;
45  ss << "{ssrcs: {";
46  for (size_t i = 0; i < ssrcs.size(); ++i) {
47    ss << ssrcs[i];
48    if (i != ssrcs.size() - 1)
49      ss << "}, {";
50  }
51  ss << '}';
52
53  ss << ", payload_type: " << payload_type;
54  ss << '}';
55  return ss.str();
56}
57
58std::string VideoSendStream::Config::Rtp::ToString() const {
59  std::stringstream ss;
60  ss << "{ssrcs: {";
61  for (size_t i = 0; i < ssrcs.size(); ++i) {
62    ss << ssrcs[i];
63    if (i != ssrcs.size() - 1)
64      ss << "}, {";
65  }
66  ss << '}';
67
68  ss << ", max_packet_size: " << max_packet_size;
69  if (min_transmit_bitrate_bps != 0)
70    ss << ", min_transmit_bitrate_bps: " << min_transmit_bitrate_bps;
71
72  ss << ", extensions: {";
73  for (size_t i = 0; i < extensions.size(); ++i) {
74    ss << extensions[i].ToString();
75    if (i != extensions.size() - 1)
76      ss << "}, {";
77  }
78  ss << '}';
79
80  if (nack.rtp_history_ms != 0)
81    ss << ", nack.rtp_history_ms: " << nack.rtp_history_ms;
82  if (fec.ulpfec_payload_type != -1 || fec.red_payload_type != -1)
83    ss << ", fec: " << fec.ToString();
84  if (rtx.payload_type != 0 || !rtx.ssrcs.empty())
85    ss << ", rtx: " << rtx.ToString();
86  if (c_name != "")
87    ss << ", c_name: " << c_name;
88  ss << '}';
89  return ss.str();
90}
91
92std::string VideoSendStream::Config::ToString() const {
93  std::stringstream ss;
94  ss << "{encoder_settings: " << encoder_settings.ToString();
95  ss << ", rtp: " << rtp.ToString();
96  if (pre_encode_callback != NULL)
97    ss << ", (pre_encode_callback)";
98  if (post_encode_callback != NULL)
99    ss << ", (post_encode_callback)";
100  if (local_renderer != NULL) {
101    ss << ", (local_renderer, render_delay_ms: " << render_delay_ms << ")";
102  }
103  if (target_delay_ms > 0)
104    ss << ", target_delay_ms: " << target_delay_ms;
105  if (suspend_below_min_bitrate)
106    ss << ", suspend_below_min_bitrate: on";
107  ss << '}';
108  return ss.str();
109}
110
111namespace internal {
112VideoSendStream::VideoSendStream(
113    newapi::Transport* transport,
114    CpuOveruseObserver* overuse_observer,
115    webrtc::VideoEngine* video_engine,
116    const VideoSendStream::Config& config,
117    const VideoEncoderConfig& encoder_config,
118    const std::map<uint32_t, RtpState>& suspended_ssrcs,
119    int base_channel,
120    int start_bitrate_bps)
121    : transport_adapter_(transport),
122      encoded_frame_proxy_(config.post_encode_callback),
123      config_(config),
124      start_bitrate_bps_(start_bitrate_bps),
125      suspended_ssrcs_(suspended_ssrcs),
126      external_codec_(NULL),
127      channel_(-1),
128      stats_proxy_(config) {
129  video_engine_base_ = ViEBase::GetInterface(video_engine);
130  video_engine_base_->CreateChannel(channel_, base_channel);
131  assert(channel_ != -1);
132  assert(start_bitrate_bps_ > 0);
133
134  rtp_rtcp_ = ViERTP_RTCP::GetInterface(video_engine);
135  assert(rtp_rtcp_ != NULL);
136
137  assert(config_.rtp.ssrcs.size() > 0);
138
139  assert(config_.rtp.min_transmit_bitrate_bps >= 0);
140  rtp_rtcp_->SetMinTransmitBitrate(channel_,
141                                   config_.rtp.min_transmit_bitrate_bps / 1000);
142
143  for (size_t i = 0; i < config_.rtp.extensions.size(); ++i) {
144    const std::string& extension = config_.rtp.extensions[i].name;
145    int id = config_.rtp.extensions[i].id;
146    if (extension == RtpExtension::kTOffset) {
147      if (rtp_rtcp_->SetSendTimestampOffsetStatus(channel_, true, id) != 0)
148        abort();
149    } else if (extension == RtpExtension::kAbsSendTime) {
150      if (rtp_rtcp_->SetSendAbsoluteSendTimeStatus(channel_, true, id) != 0)
151        abort();
152    } else {
153      abort();  // Unsupported extension.
154    }
155  }
156
157  rtp_rtcp_->SetRembStatus(channel_, true, false);
158
159  // Enable NACK, FEC or both.
160  if (config_.rtp.fec.red_payload_type != -1) {
161    assert(config_.rtp.fec.ulpfec_payload_type != -1);
162    if (config_.rtp.nack.rtp_history_ms > 0) {
163      rtp_rtcp_->SetHybridNACKFECStatus(
164          channel_,
165          true,
166          static_cast<unsigned char>(config_.rtp.fec.red_payload_type),
167          static_cast<unsigned char>(config_.rtp.fec.ulpfec_payload_type));
168    } else {
169      rtp_rtcp_->SetFECStatus(
170          channel_,
171          true,
172          static_cast<unsigned char>(config_.rtp.fec.red_payload_type),
173          static_cast<unsigned char>(config_.rtp.fec.ulpfec_payload_type));
174    }
175  } else {
176    rtp_rtcp_->SetNACKStatus(channel_, config_.rtp.nack.rtp_history_ms > 0);
177  }
178
179  ConfigureSsrcs();
180
181  char rtcp_cname[ViERTP_RTCP::KMaxRTCPCNameLength];
182  assert(config_.rtp.c_name.length() < ViERTP_RTCP::KMaxRTCPCNameLength);
183  strncpy(rtcp_cname, config_.rtp.c_name.c_str(), sizeof(rtcp_cname) - 1);
184  rtcp_cname[sizeof(rtcp_cname) - 1] = '\0';
185
186  rtp_rtcp_->SetRTCPCName(channel_, rtcp_cname);
187
188  capture_ = ViECapture::GetInterface(video_engine);
189  capture_->AllocateExternalCaptureDevice(capture_id_, external_capture_);
190  capture_->ConnectCaptureDevice(capture_id_, channel_);
191
192  network_ = ViENetwork::GetInterface(video_engine);
193  assert(network_ != NULL);
194
195  network_->RegisterSendTransport(channel_, transport_adapter_);
196  // 28 to match packet overhead in ModuleRtpRtcpImpl.
197  network_->SetMTU(channel_,
198                   static_cast<unsigned int>(config_.rtp.max_packet_size + 28));
199
200  assert(config.encoder_settings.encoder != NULL);
201  assert(config.encoder_settings.payload_type >= 0);
202  assert(config.encoder_settings.payload_type <= 127);
203  external_codec_ = ViEExternalCodec::GetInterface(video_engine);
204  if (external_codec_->RegisterExternalSendCodec(
205          channel_,
206          config.encoder_settings.payload_type,
207          config.encoder_settings.encoder,
208          false) != 0) {
209    abort();
210  }
211
212  codec_ = ViECodec::GetInterface(video_engine);
213  if (!ReconfigureVideoEncoder(encoder_config))
214    abort();
215
216  if (overuse_observer)
217    video_engine_base_->RegisterCpuOveruseObserver(channel_, overuse_observer);
218
219  video_engine_base_->RegisterSendSideDelayObserver(channel_, &stats_proxy_);
220
221  image_process_ = ViEImageProcess::GetInterface(video_engine);
222  image_process_->RegisterPreEncodeCallback(channel_,
223                                            config_.pre_encode_callback);
224  if (config_.post_encode_callback) {
225    image_process_->RegisterPostEncodeImageCallback(channel_,
226                                                    &encoded_frame_proxy_);
227  }
228
229  if (config_.suspend_below_min_bitrate)
230    codec_->SuspendBelowMinBitrate(channel_);
231
232  rtp_rtcp_->RegisterSendChannelRtcpStatisticsCallback(channel_,
233                                                       &stats_proxy_);
234  rtp_rtcp_->RegisterSendChannelRtpStatisticsCallback(channel_,
235                                                      &stats_proxy_);
236  rtp_rtcp_->RegisterSendBitrateObserver(channel_, &stats_proxy_);
237  rtp_rtcp_->RegisterSendFrameCountObserver(channel_, &stats_proxy_);
238
239  codec_->RegisterEncoderObserver(channel_, stats_proxy_);
240  capture_->RegisterObserver(capture_id_, stats_proxy_);
241}
242
243VideoSendStream::~VideoSendStream() {
244  capture_->DeregisterObserver(capture_id_);
245  codec_->DeregisterEncoderObserver(channel_);
246
247  rtp_rtcp_->DeregisterSendFrameCountObserver(channel_, &stats_proxy_);
248  rtp_rtcp_->DeregisterSendBitrateObserver(channel_, &stats_proxy_);
249  rtp_rtcp_->DeregisterSendChannelRtpStatisticsCallback(channel_,
250                                                        &stats_proxy_);
251  rtp_rtcp_->DeregisterSendChannelRtcpStatisticsCallback(channel_,
252                                                         &stats_proxy_);
253
254  image_process_->DeRegisterPreEncodeCallback(channel_);
255
256  network_->DeregisterSendTransport(channel_);
257
258  capture_->DisconnectCaptureDevice(channel_);
259  capture_->ReleaseCaptureDevice(capture_id_);
260
261  external_codec_->DeRegisterExternalSendCodec(
262      channel_, config_.encoder_settings.payload_type);
263
264  video_engine_base_->DeleteChannel(channel_);
265
266  image_process_->Release();
267  video_engine_base_->Release();
268  capture_->Release();
269  codec_->Release();
270  if (external_codec_)
271    external_codec_->Release();
272  network_->Release();
273  rtp_rtcp_->Release();
274}
275
276void VideoSendStream::SwapFrame(I420VideoFrame* frame) {
277  // TODO(pbos): Local rendering should not be done on the capture thread.
278  if (config_.local_renderer != NULL)
279    config_.local_renderer->RenderFrame(*frame, 0);
280
281  external_capture_->SwapFrame(frame);
282}
283
284VideoSendStreamInput* VideoSendStream::Input() { return this; }
285
286void VideoSendStream::Start() {
287  transport_adapter_.Enable();
288  video_engine_base_->StartSend(channel_);
289  video_engine_base_->StartReceive(channel_);
290}
291
292void VideoSendStream::Stop() {
293  video_engine_base_->StopSend(channel_);
294  video_engine_base_->StopReceive(channel_);
295  transport_adapter_.Disable();
296}
297
298bool VideoSendStream::ReconfigureVideoEncoder(
299    const VideoEncoderConfig& config) {
300  const std::vector<VideoStream>& streams = config.streams;
301  assert(!streams.empty());
302  assert(config_.rtp.ssrcs.size() >= streams.size());
303
304  VideoCodec video_codec;
305  memset(&video_codec, 0, sizeof(video_codec));
306  if (config_.encoder_settings.payload_name == "VP8") {
307    video_codec.codecType = kVideoCodecVP8;
308  } else if (config_.encoder_settings.payload_name == "H264") {
309    video_codec.codecType = kVideoCodecH264;
310  } else {
311    video_codec.codecType = kVideoCodecGeneric;
312  }
313  switch (config.content_type) {
314    case VideoEncoderConfig::kRealtimeVideo:
315      video_codec.mode = kRealtimeVideo;
316      break;
317    case VideoEncoderConfig::kScreenshare:
318      video_codec.mode = kScreensharing;
319      break;
320  }
321
322  if (video_codec.codecType == kVideoCodecVP8) {
323    video_codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings();
324  } else if (video_codec.codecType == kVideoCodecH264) {
325    video_codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings();
326  }
327
328  if (video_codec.codecType == kVideoCodecVP8) {
329    if (config.encoder_specific_settings != NULL) {
330      video_codec.codecSpecific.VP8 = *reinterpret_cast<const VideoCodecVP8*>(
331                                          config.encoder_specific_settings);
332    }
333    video_codec.codecSpecific.VP8.numberOfTemporalLayers =
334        static_cast<unsigned char>(streams.back().temporal_layers.size());
335  } else {
336    // TODO(pbos): Support encoder_settings codec-agnostically.
337    assert(config.encoder_specific_settings == NULL);
338  }
339
340  strncpy(video_codec.plName,
341          config_.encoder_settings.payload_name.c_str(),
342          kPayloadNameSize - 1);
343  video_codec.plName[kPayloadNameSize - 1] = '\0';
344  video_codec.plType = config_.encoder_settings.payload_type;
345  video_codec.numberOfSimulcastStreams =
346      static_cast<unsigned char>(streams.size());
347  video_codec.minBitrate = streams[0].min_bitrate_bps / 1000;
348  assert(streams.size() <= kMaxSimulcastStreams);
349  for (size_t i = 0; i < streams.size(); ++i) {
350    SimulcastStream* sim_stream = &video_codec.simulcastStream[i];
351    assert(streams[i].width > 0);
352    assert(streams[i].height > 0);
353    assert(streams[i].max_framerate > 0);
354    // Different framerates not supported per stream at the moment.
355    assert(streams[i].max_framerate == streams[0].max_framerate);
356    assert(streams[i].min_bitrate_bps >= 0);
357    assert(streams[i].target_bitrate_bps >= streams[i].min_bitrate_bps);
358    assert(streams[i].max_bitrate_bps >= streams[i].target_bitrate_bps);
359    assert(streams[i].max_qp >= 0);
360
361    sim_stream->width = static_cast<unsigned short>(streams[i].width);
362    sim_stream->height = static_cast<unsigned short>(streams[i].height);
363    sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000;
364    sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000;
365    sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000;
366    sim_stream->qpMax = streams[i].max_qp;
367    sim_stream->numberOfTemporalLayers =
368        static_cast<unsigned char>(streams[i].temporal_layers.size());
369
370    video_codec.width = std::max(video_codec.width,
371                                 static_cast<unsigned short>(streams[i].width));
372    video_codec.height = std::max(
373        video_codec.height, static_cast<unsigned short>(streams[i].height));
374    video_codec.minBitrate =
375        std::min(video_codec.minBitrate,
376                 static_cast<unsigned int>(streams[i].min_bitrate_bps / 1000));
377    video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000;
378    video_codec.qpMax = std::max(video_codec.qpMax,
379                                 static_cast<unsigned int>(streams[i].max_qp));
380  }
381  video_codec.startBitrate =
382      static_cast<unsigned int>(start_bitrate_bps_) / 1000;
383
384  if (video_codec.minBitrate < kViEMinCodecBitrate)
385    video_codec.minBitrate = kViEMinCodecBitrate;
386  if (video_codec.maxBitrate < kViEMinCodecBitrate)
387    video_codec.maxBitrate = kViEMinCodecBitrate;
388  if (video_codec.startBitrate < video_codec.minBitrate)
389    video_codec.startBitrate = video_codec.minBitrate;
390  if (video_codec.startBitrate > video_codec.maxBitrate)
391    video_codec.startBitrate = video_codec.maxBitrate;
392
393  if (video_codec.startBitrate < video_codec.minBitrate)
394    video_codec.startBitrate = video_codec.minBitrate;
395  if (video_codec.startBitrate > video_codec.maxBitrate)
396    video_codec.startBitrate = video_codec.maxBitrate;
397
398  assert(streams[0].max_framerate > 0);
399  video_codec.maxFramerate = streams[0].max_framerate;
400
401  return codec_->SetSendCodec(channel_, video_codec) == 0;
402}
403
404bool VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
405  return network_->ReceivedRTCPPacket(
406             channel_, packet, static_cast<int>(length)) == 0;
407}
408
409VideoSendStream::Stats VideoSendStream::GetStats() const {
410  return stats_proxy_.GetStats();
411}
412
413void VideoSendStream::ConfigureSsrcs() {
414  for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
415    uint32_t ssrc = config_.rtp.ssrcs[i];
416    rtp_rtcp_->SetLocalSSRC(
417        channel_, ssrc, kViEStreamTypeNormal, static_cast<unsigned char>(i));
418    RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
419    if (it != suspended_ssrcs_.end())
420      rtp_rtcp_->SetRtpStateForSsrc(channel_, ssrc, it->second);
421  }
422
423  if (config_.rtp.rtx.ssrcs.empty()) {
424    assert(!config_.rtp.rtx.pad_with_redundant_payloads);
425    return;
426  }
427
428  // Set up RTX.
429  assert(config_.rtp.rtx.ssrcs.size() == config_.rtp.ssrcs.size());
430  for (size_t i = 0; i < config_.rtp.rtx.ssrcs.size(); ++i) {
431    uint32_t ssrc = config_.rtp.rtx.ssrcs[i];
432    rtp_rtcp_->SetLocalSSRC(channel_,
433                            config_.rtp.rtx.ssrcs[i],
434                            kViEStreamTypeRtx,
435                            static_cast<unsigned char>(i));
436    RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
437    if (it != suspended_ssrcs_.end())
438      rtp_rtcp_->SetRtpStateForSsrc(channel_, ssrc, it->second);
439  }
440
441  if (config_.rtp.rtx.pad_with_redundant_payloads) {
442    rtp_rtcp_->SetPadWithRedundantPayloads(channel_, true);
443  }
444
445  assert(config_.rtp.rtx.payload_type >= 0);
446  rtp_rtcp_->SetRtxSendPayloadType(channel_, config_.rtp.rtx.payload_type);
447}
448
449std::map<uint32_t, RtpState> VideoSendStream::GetRtpStates() const {
450  std::map<uint32_t, RtpState> rtp_states;
451  for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
452    uint32_t ssrc = config_.rtp.ssrcs[i];
453    rtp_states[ssrc] = rtp_rtcp_->GetRtpStateForSsrc(channel_, ssrc);
454  }
455
456  for (size_t i = 0; i < config_.rtp.rtx.ssrcs.size(); ++i) {
457    uint32_t ssrc = config_.rtp.rtx.ssrcs[i];
458    rtp_states[ssrc] = rtp_rtcp_->GetRtpStateForSsrc(channel_, ssrc);
459  }
460
461  return rtp_states;
462}
463
464void VideoSendStream::SignalNetworkState(Call::NetworkState state) {
465  // When network goes up, enable RTCP status before setting transmission state.
466  // When it goes down, disable RTCP afterwards. This ensures that any packets
467  // sent due to the network state changed will not be dropped.
468  if (state == Call::kNetworkUp)
469    rtp_rtcp_->SetRTCPStatus(channel_, kRtcpCompound_RFC4585);
470  network_->SetNetworkTransmissionState(channel_, state == Call::kNetworkUp);
471  if (state == Call::kNetworkDown)
472    rtp_rtcp_->SetRTCPStatus(channel_, kRtcpNone);
473}
474
475}  // namespace internal
476}  // namespace webrtc
477