1/*
2 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/common_types.h"
12
13#include <algorithm>  // std::max
14
15#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
16#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
17#include "webrtc/modules/video_coding/main/source/encoded_frame.h"
18#include "webrtc/modules/video_coding/main/source/video_coding_impl.h"
19#include "webrtc/system_wrappers/interface/clock.h"
20#include "webrtc/system_wrappers/interface/logging.h"
21
22namespace webrtc {
23namespace vcm {
24
25class DebugRecorder {
26 public:
27  DebugRecorder()
28      : cs_(CriticalSectionWrapper::CreateCriticalSection()), file_(NULL) {}
29
30  ~DebugRecorder() { Stop(); }
31
32  int Start(const char* file_name_utf8) {
33    CriticalSectionScoped cs(cs_.get());
34    if (file_)
35      fclose(file_);
36    file_ = fopen(file_name_utf8, "wb");
37    if (!file_)
38      return VCM_GENERAL_ERROR;
39    return VCM_OK;
40  }
41
42  void Stop() {
43    CriticalSectionScoped cs(cs_.get());
44    if (file_) {
45      fclose(file_);
46      file_ = NULL;
47    }
48  }
49
50  void Add(const I420VideoFrame& frame) {
51    CriticalSectionScoped cs(cs_.get());
52    if (file_)
53      PrintI420VideoFrame(frame, file_);
54  }
55
56 private:
57  scoped_ptr<CriticalSectionWrapper> cs_;
58  FILE* file_ GUARDED_BY(cs_);
59};
60
61VideoSender::VideoSender(Clock* clock,
62                         EncodedImageCallback* post_encode_callback)
63    : clock_(clock),
64      recorder_(new DebugRecorder()),
65      process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
66      _sendCritSect(CriticalSectionWrapper::CreateCriticalSection()),
67      _encoder(),
68      _encodedFrameCallback(post_encode_callback),
69      _nextFrameTypes(1, kVideoFrameDelta),
70      _mediaOpt(clock_),
71      _sendStatsCallback(NULL),
72      _codecDataBase(),
73      frame_dropper_enabled_(true),
74      _sendStatsTimer(1000, clock_),
75      qm_settings_callback_(NULL),
76      protection_callback_(NULL) {}
77
78VideoSender::~VideoSender() {
79  delete _sendCritSect;
80}
81
82int32_t VideoSender::Process() {
83  int32_t returnValue = VCM_OK;
84
85  if (_sendStatsTimer.TimeUntilProcess() == 0) {
86    _sendStatsTimer.Processed();
87    CriticalSectionScoped cs(process_crit_sect_.get());
88    if (_sendStatsCallback != NULL) {
89      uint32_t bitRate;
90      uint32_t frameRate;
91      {
92        CriticalSectionScoped cs(_sendCritSect);
93        bitRate = _mediaOpt.SentBitRate();
94        frameRate = _mediaOpt.SentFrameRate();
95      }
96      _sendStatsCallback->SendStatistics(bitRate, frameRate);
97    }
98  }
99
100  return returnValue;
101}
102
103// Reset send side to initial state - all components
104int32_t VideoSender::InitializeSender() {
105  CriticalSectionScoped cs(_sendCritSect);
106  _codecDataBase.ResetSender();
107  _encoder = NULL;
108  _encodedFrameCallback.SetTransportCallback(NULL);
109  _mediaOpt.Reset();  // Resetting frame dropper
110  return VCM_OK;
111}
112
113int32_t VideoSender::TimeUntilNextProcess() {
114  return _sendStatsTimer.TimeUntilProcess();
115}
116
117// Register the send codec to be used.
118int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec,
119                                       uint32_t numberOfCores,
120                                       uint32_t maxPayloadSize) {
121  CriticalSectionScoped cs(_sendCritSect);
122  if (sendCodec == NULL) {
123    return VCM_PARAMETER_ERROR;
124  }
125
126  bool ret = _codecDataBase.SetSendCodec(
127      sendCodec, numberOfCores, maxPayloadSize, &_encodedFrameCallback);
128
129  // Update encoder regardless of result to make sure that we're not holding on
130  // to a deleted instance.
131  _encoder = _codecDataBase.GetEncoder();
132
133  if (!ret) {
134    LOG(LS_ERROR) << "Failed to initialize the encoder with payload name "
135                  << sendCodec->plName << ". Error code: " << ret;
136    return VCM_CODEC_ERROR;
137  }
138
139  int numLayers = (sendCodec->codecType != kVideoCodecVP8)
140                      ? 1
141                      : sendCodec->codecSpecific.VP8.numberOfTemporalLayers;
142  // If we have screensharing and we have layers, we disable frame dropper.
143  bool disable_frame_dropper =
144      numLayers > 1 && sendCodec->mode == kScreensharing;
145  if (disable_frame_dropper) {
146    _mediaOpt.EnableFrameDropper(false);
147  } else if (frame_dropper_enabled_) {
148    _mediaOpt.EnableFrameDropper(true);
149  }
150  _nextFrameTypes.clear();
151  _nextFrameTypes.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1),
152                         kVideoFrameDelta);
153
154  _mediaOpt.SetEncodingData(sendCodec->codecType,
155                            sendCodec->maxBitrate * 1000,
156                            sendCodec->maxFramerate * 1000,
157                            sendCodec->startBitrate * 1000,
158                            sendCodec->width,
159                            sendCodec->height,
160                            numLayers,
161                            maxPayloadSize);
162  return VCM_OK;
163}
164
165// Get current send codec
166int32_t VideoSender::SendCodec(VideoCodec* currentSendCodec) const {
167  CriticalSectionScoped cs(_sendCritSect);
168
169  if (currentSendCodec == NULL) {
170    return VCM_PARAMETER_ERROR;
171  }
172  return _codecDataBase.SendCodec(currentSendCodec) ? 0 : -1;
173}
174
175// Get the current send codec type
176VideoCodecType VideoSender::SendCodec() const {
177  CriticalSectionScoped cs(_sendCritSect);
178
179  return _codecDataBase.SendCodec();
180}
181
182// Register an external decoder object.
183// This can not be used together with external decoder callbacks.
184int32_t VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder,
185                                             uint8_t payloadType,
186                                             bool internalSource /*= false*/) {
187  CriticalSectionScoped cs(_sendCritSect);
188
189  if (externalEncoder == NULL) {
190    bool wasSendCodec = false;
191    const bool ret =
192        _codecDataBase.DeregisterExternalEncoder(payloadType, &wasSendCodec);
193    if (wasSendCodec) {
194      // Make sure the VCM doesn't use the de-registered codec
195      _encoder = NULL;
196    }
197    return ret ? 0 : -1;
198  }
199  _codecDataBase.RegisterExternalEncoder(
200      externalEncoder, payloadType, internalSource);
201  return 0;
202}
203
204// Get codec config parameters
205int32_t VideoSender::CodecConfigParameters(uint8_t* buffer,
206                                           int32_t size) const {
207  CriticalSectionScoped cs(_sendCritSect);
208  if (_encoder != NULL) {
209    return _encoder->CodecConfigParameters(buffer, size);
210  }
211  return VCM_UNINITIALIZED;
212}
213
214// TODO(andresp): Make const once media_opt is thread-safe and this has a
215// pointer to it.
216int32_t VideoSender::SentFrameCount(VCMFrameCount* frameCount) {
217  CriticalSectionScoped cs(_sendCritSect);
218  *frameCount = _mediaOpt.SentFrameCount();
219  return VCM_OK;
220}
221
222// Get encode bitrate
223int VideoSender::Bitrate(unsigned int* bitrate) const {
224  CriticalSectionScoped cs(_sendCritSect);
225  // return the bit rate which the encoder is set to
226  if (!_encoder) {
227    return VCM_UNINITIALIZED;
228  }
229  *bitrate = _encoder->BitRate();
230  return 0;
231}
232
233// Get encode frame rate
234int VideoSender::FrameRate(unsigned int* framerate) const {
235  CriticalSectionScoped cs(_sendCritSect);
236  // input frame rate, not compensated
237  if (!_encoder) {
238    return VCM_UNINITIALIZED;
239  }
240  *framerate = _encoder->FrameRate();
241  return 0;
242}
243
244// Set channel parameters
245int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate,
246                                          uint8_t lossRate,
247                                          uint32_t rtt) {
248  int32_t ret = 0;
249  {
250    CriticalSectionScoped sendCs(_sendCritSect);
251    uint32_t targetRate = _mediaOpt.SetTargetRates(target_bitrate,
252                                                   lossRate,
253                                                   rtt,
254                                                   protection_callback_,
255                                                   qm_settings_callback_);
256    if (_encoder != NULL) {
257      ret = _encoder->SetChannelParameters(lossRate, rtt);
258      if (ret < 0) {
259        return ret;
260      }
261      ret = (int32_t)_encoder->SetRates(targetRate, _mediaOpt.InputFrameRate());
262      if (ret < 0) {
263        return ret;
264      }
265    } else {
266      return VCM_UNINITIALIZED;
267    }  // encoder
268  }    // send side
269  return VCM_OK;
270}
271
272int32_t VideoSender::RegisterTransportCallback(
273    VCMPacketizationCallback* transport) {
274  CriticalSectionScoped cs(_sendCritSect);
275  _encodedFrameCallback.SetMediaOpt(&_mediaOpt);
276  _encodedFrameCallback.SetTransportCallback(transport);
277  return VCM_OK;
278}
279
280// Register video output information callback which will be called to deliver
281// information about the video stream produced by the encoder, for instance the
282// average frame rate and bit rate.
283int32_t VideoSender::RegisterSendStatisticsCallback(
284    VCMSendStatisticsCallback* sendStats) {
285  CriticalSectionScoped cs(process_crit_sect_.get());
286  _sendStatsCallback = sendStats;
287  return VCM_OK;
288}
289
290// Register a video quality settings callback which will be called when frame
291// rate/dimensions need to be updated for video quality optimization
292int32_t VideoSender::RegisterVideoQMCallback(
293    VCMQMSettingsCallback* qm_settings_callback) {
294  CriticalSectionScoped cs(_sendCritSect);
295  qm_settings_callback_ = qm_settings_callback;
296  _mediaOpt.EnableQM(qm_settings_callback_ != NULL);
297  return VCM_OK;
298}
299
300// Register a video protection callback which will be called to deliver the
301// requested FEC rate and NACK status (on/off).
302int32_t VideoSender::RegisterProtectionCallback(
303    VCMProtectionCallback* protection_callback) {
304  CriticalSectionScoped cs(_sendCritSect);
305  protection_callback_ = protection_callback;
306  return VCM_OK;
307}
308
309// Enable or disable a video protection method.
310// Note: This API should be deprecated, as it does not offer a distinction
311// between the protection method and decoding with or without errors. If such a
312// behavior is desired, use the following API: SetReceiverRobustnessMode.
313int32_t VideoSender::SetVideoProtection(VCMVideoProtection videoProtection,
314                                        bool enable) {
315  switch (videoProtection) {
316    case kProtectionNack:
317    case kProtectionNackSender: {
318      CriticalSectionScoped cs(_sendCritSect);
319      _mediaOpt.EnableProtectionMethod(enable, media_optimization::kNack);
320      break;
321    }
322
323    case kProtectionNackFEC: {
324      CriticalSectionScoped cs(_sendCritSect);
325      _mediaOpt.EnableProtectionMethod(enable, media_optimization::kNackFec);
326      break;
327    }
328
329    case kProtectionFEC: {
330      CriticalSectionScoped cs(_sendCritSect);
331      _mediaOpt.EnableProtectionMethod(enable, media_optimization::kFec);
332      break;
333    }
334
335    case kProtectionPeriodicKeyFrames: {
336      CriticalSectionScoped cs(_sendCritSect);
337      return _codecDataBase.SetPeriodicKeyFrames(enable) ? 0 : -1;
338      break;
339    }
340    case kProtectionNackReceiver:
341    case kProtectionDualDecoder:
342    case kProtectionKeyOnLoss:
343    case kProtectionKeyOnKeyLoss:
344      // Ignore decoder modes.
345      return VCM_OK;
346  }
347  return VCM_OK;
348}
349// Add one raw video frame to the encoder, blocking.
350int32_t VideoSender::AddVideoFrame(const I420VideoFrame& videoFrame,
351                                   const VideoContentMetrics* contentMetrics,
352                                   const CodecSpecificInfo* codecSpecificInfo) {
353  CriticalSectionScoped cs(_sendCritSect);
354  if (_encoder == NULL) {
355    return VCM_UNINITIALIZED;
356  }
357  // TODO(holmer): Add support for dropping frames per stream. Currently we
358  // only have one frame dropper for all streams.
359  if (_nextFrameTypes[0] == kFrameEmpty) {
360    return VCM_OK;
361  }
362  if (_mediaOpt.DropFrame()) {
363    return VCM_OK;
364  }
365  _mediaOpt.UpdateContentData(contentMetrics);
366  int32_t ret =
367      _encoder->Encode(videoFrame, codecSpecificInfo, _nextFrameTypes);
368  recorder_->Add(videoFrame);
369  if (ret < 0) {
370    LOG(LS_ERROR) << "Failed to encode frame. Error code: " << ret;
371    return ret;
372  }
373  for (size_t i = 0; i < _nextFrameTypes.size(); ++i) {
374    _nextFrameTypes[i] = kVideoFrameDelta;  // Default frame type.
375  }
376  return VCM_OK;
377}
378
379int32_t VideoSender::IntraFrameRequest(int stream_index) {
380  CriticalSectionScoped cs(_sendCritSect);
381  if (stream_index < 0 ||
382      static_cast<unsigned int>(stream_index) >= _nextFrameTypes.size()) {
383    return -1;
384  }
385  _nextFrameTypes[stream_index] = kVideoFrameKey;
386  if (_encoder != NULL && _encoder->InternalSource()) {
387    // Try to request the frame if we have an external encoder with
388    // internal source since AddVideoFrame never will be called.
389    if (_encoder->RequestFrame(_nextFrameTypes) == WEBRTC_VIDEO_CODEC_OK) {
390      _nextFrameTypes[stream_index] = kVideoFrameDelta;
391    }
392  }
393  return VCM_OK;
394}
395
396int32_t VideoSender::EnableFrameDropper(bool enable) {
397  CriticalSectionScoped cs(_sendCritSect);
398  frame_dropper_enabled_ = enable;
399  _mediaOpt.EnableFrameDropper(enable);
400  return VCM_OK;
401}
402
403int VideoSender::SetSenderNackMode(SenderNackMode mode) {
404  CriticalSectionScoped cs(_sendCritSect);
405
406  switch (mode) {
407    case VideoCodingModule::kNackNone:
408      _mediaOpt.EnableProtectionMethod(false, media_optimization::kNack);
409      break;
410    case VideoCodingModule::kNackAll:
411      _mediaOpt.EnableProtectionMethod(true, media_optimization::kNack);
412      break;
413    case VideoCodingModule::kNackSelective:
414      return VCM_NOT_IMPLEMENTED;
415      break;
416  }
417  return VCM_OK;
418}
419
420int VideoSender::SetSenderReferenceSelection(bool enable) {
421  return VCM_NOT_IMPLEMENTED;
422}
423
424int VideoSender::SetSenderFEC(bool enable) {
425  CriticalSectionScoped cs(_sendCritSect);
426  _mediaOpt.EnableProtectionMethod(enable, media_optimization::kFec);
427  return VCM_OK;
428}
429
430int VideoSender::SetSenderKeyFramePeriod(int periodMs) {
431  return VCM_NOT_IMPLEMENTED;
432}
433
434int VideoSender::StartDebugRecording(const char* file_name_utf8) {
435  return recorder_->Start(file_name_utf8);
436}
437
438void VideoSender::StopDebugRecording() {
439  recorder_->Stop();
440}
441
442void VideoSender::SuspendBelowMinBitrate() {
443  CriticalSectionScoped cs(_sendCritSect);
444  VideoCodec current_send_codec;
445  if (SendCodec(&current_send_codec) != 0) {
446    assert(false);  // Must set a send codec before SuspendBelowMinBitrate.
447    return;
448  }
449  int threshold_bps;
450  if (current_send_codec.numberOfSimulcastStreams == 0) {
451    threshold_bps = current_send_codec.minBitrate * 1000;
452  } else {
453    threshold_bps = current_send_codec.simulcastStream[0].minBitrate * 1000;
454  }
455  // Set the hysteresis window to be at 10% of the threshold, but at least
456  // 10 kbps.
457  int window_bps = std::max(threshold_bps / 10, 10000);
458  _mediaOpt.SuspendBelowMinBitrate(threshold_bps, window_bps);
459}
460
461bool VideoSender::VideoSuspended() const {
462  CriticalSectionScoped cs(_sendCritSect);
463  return _mediaOpt.IsVideoSuspended();
464}
465}  // namespace vcm
466}  // namespace webrtc
467