1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/video_engine/vie_encoder.h"
12
13#include <assert.h>
14
15#include <algorithm>
16
17#include "webrtc/common_video/interface/video_image.h"
18#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
19#include "webrtc/modules/pacing/include/paced_sender.h"
20#include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h"
21#include "webrtc/modules/utility/interface/process_thread.h"
22#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
23#include "webrtc/modules/video_coding/main/interface/video_coding.h"
24#include "webrtc/modules/video_coding/main/interface/video_coding_defines.h"
25#include "webrtc/modules/video_coding/main/source/encoded_frame.h"
26#include "webrtc/system_wrappers/interface/clock.h"
27#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
28#include "webrtc/system_wrappers/interface/logging.h"
29#include "webrtc/system_wrappers/interface/tick_util.h"
30#include "webrtc/system_wrappers/interface/trace_event.h"
31#include "webrtc/video_engine/include/vie_codec.h"
32#include "webrtc/video_engine/include/vie_image_process.h"
33#include "webrtc/frame_callback.h"
34#include "webrtc/video_engine/vie_defines.h"
35
36namespace webrtc {
37
38// Margin on when we pause the encoder when the pacing buffer overflows relative
39// to the configured buffer delay.
40static const float kEncoderPausePacerMargin = 2.0f;
41
42// Don't stop the encoder unless the delay is above this configured value.
43static const int kMinPacingDelayMs = 200;
44
45// Allow packets to be transmitted in up to 2 times max video bitrate if the
46// bandwidth estimate allows it.
47// TODO(holmer): Expose transmission start, min and max bitrates in the
48// VideoEngine API and remove the kTransmissionMaxBitrateMultiplier.
49static const int kTransmissionMaxBitrateMultiplier = 2;
50
51static const float kStopPaddingThresholdMs = 2000;
52
53std::vector<uint32_t> AllocateStreamBitrates(
54    uint32_t total_bitrate,
55    const SimulcastStream* stream_configs,
56    size_t number_of_streams) {
57  if (number_of_streams == 0) {
58    std::vector<uint32_t> stream_bitrates(1, 0);
59    stream_bitrates[0] = total_bitrate;
60    return stream_bitrates;
61  }
62  std::vector<uint32_t> stream_bitrates(number_of_streams, 0);
63  uint32_t bitrate_remainder = total_bitrate;
64  for (size_t i = 0; i < stream_bitrates.size() && bitrate_remainder > 0; ++i) {
65    if (stream_configs[i].maxBitrate * 1000 > bitrate_remainder) {
66      stream_bitrates[i] = bitrate_remainder;
67    } else {
68      stream_bitrates[i] = stream_configs[i].maxBitrate * 1000;
69    }
70    bitrate_remainder -= stream_bitrates[i];
71  }
72  return stream_bitrates;
73}
74
75class QMVideoSettingsCallback : public VCMQMSettingsCallback {
76 public:
77  explicit QMVideoSettingsCallback(VideoProcessingModule* vpm);
78
79  ~QMVideoSettingsCallback();
80
81  // Update VPM with QM (quality modes: frame size & frame rate) settings.
82  int32_t SetVideoQMSettings(const uint32_t frame_rate,
83                             const uint32_t width,
84                             const uint32_t height);
85
86 private:
87  VideoProcessingModule* vpm_;
88};
89
90class ViEBitrateObserver : public BitrateObserver {
91 public:
92  explicit ViEBitrateObserver(ViEEncoder* owner)
93      : owner_(owner) {
94  }
95  virtual ~ViEBitrateObserver() {}
96  // Implements BitrateObserver.
97  virtual void OnNetworkChanged(const uint32_t bitrate_bps,
98                                const uint8_t fraction_lost,
99                                const uint32_t rtt) {
100    owner_->OnNetworkChanged(bitrate_bps, fraction_lost, rtt);
101  }
102 private:
103  ViEEncoder* owner_;
104};
105
106class ViEPacedSenderCallback : public PacedSender::Callback {
107 public:
108  explicit ViEPacedSenderCallback(ViEEncoder* owner)
109      : owner_(owner) {
110  }
111  virtual ~ViEPacedSenderCallback() {}
112  virtual bool TimeToSendPacket(uint32_t ssrc, uint16_t sequence_number,
113                                int64_t capture_time_ms, bool retransmission) {
114    return owner_->TimeToSendPacket(ssrc, sequence_number, capture_time_ms,
115                                    retransmission);
116  }
117  virtual int TimeToSendPadding(int bytes) {
118    return owner_->TimeToSendPadding(bytes);
119  }
120 private:
121  ViEEncoder* owner_;
122};
123
124ViEEncoder::ViEEncoder(int32_t engine_id,
125                       int32_t channel_id,
126                       uint32_t number_of_cores,
127                       const Config& config,
128                       ProcessThread& module_process_thread,
129                       BitrateController* bitrate_controller)
130  : engine_id_(engine_id),
131    channel_id_(channel_id),
132    number_of_cores_(number_of_cores),
133    vcm_(*webrtc::VideoCodingModule::Create()),
134    vpm_(*webrtc::VideoProcessingModule::Create(ViEModuleId(engine_id,
135                                                            channel_id))),
136    callback_cs_(CriticalSectionWrapper::CreateCriticalSection()),
137    data_cs_(CriticalSectionWrapper::CreateCriticalSection()),
138    bitrate_controller_(bitrate_controller),
139    time_of_last_incoming_frame_ms_(0),
140    send_padding_(false),
141    min_transmit_bitrate_kbps_(0),
142    target_delay_ms_(0),
143    network_is_transmitting_(true),
144    encoder_paused_(false),
145    encoder_paused_and_dropped_frame_(false),
146    fec_enabled_(false),
147    nack_enabled_(false),
148    codec_observer_(NULL),
149    effect_filter_(NULL),
150    module_process_thread_(module_process_thread),
151    has_received_sli_(false),
152    picture_id_sli_(0),
153    has_received_rpsi_(false),
154    picture_id_rpsi_(0),
155    qm_callback_(NULL),
156    video_suspended_(false),
157    pre_encode_callback_(NULL) {
158  RtpRtcp::Configuration configuration;
159  configuration.id = ViEModuleId(engine_id_, channel_id_);
160  configuration.audio = false;  // Video.
161
162  default_rtp_rtcp_.reset(RtpRtcp::CreateRtpRtcp(configuration));
163  bitrate_observer_.reset(new ViEBitrateObserver(this));
164  pacing_callback_.reset(new ViEPacedSenderCallback(this));
165  paced_sender_.reset(
166      new PacedSender(Clock::GetRealTimeClock(), pacing_callback_.get(),
167                      PacedSender::kDefaultInitialPaceKbps, 0));
168}
169
170bool ViEEncoder::Init() {
171  if (vcm_.InitializeSender() != 0) {
172    return false;
173  }
174  vpm_.EnableTemporalDecimation(true);
175
176  // Enable/disable content analysis: off by default for now.
177  vpm_.EnableContentAnalysis(false);
178
179  if (module_process_thread_.RegisterModule(&vcm_) != 0 ||
180      module_process_thread_.RegisterModule(default_rtp_rtcp_.get()) != 0 ||
181      module_process_thread_.RegisterModule(paced_sender_.get()) != 0) {
182    return false;
183  }
184  if (qm_callback_) {
185    delete qm_callback_;
186  }
187  qm_callback_ = new QMVideoSettingsCallback(&vpm_);
188
189#ifdef VIDEOCODEC_VP8
190  VideoCodecType codec_type = webrtc::kVideoCodecVP8;
191#else
192  VideoCodecType codec_type = webrtc::kVideoCodecI420;
193#endif
194
195  VideoCodec video_codec;
196  if (vcm_.Codec(codec_type, &video_codec) != VCM_OK) {
197    return false;
198  }
199  {
200    CriticalSectionScoped cs(data_cs_.get());
201    send_padding_ = video_codec.numberOfSimulcastStreams > 1;
202  }
203  if (vcm_.RegisterSendCodec(&video_codec, number_of_cores_,
204                             default_rtp_rtcp_->MaxDataPayloadLength()) != 0) {
205    return false;
206  }
207  if (default_rtp_rtcp_->RegisterSendPayload(video_codec) != 0) {
208    return false;
209  }
210  if (vcm_.RegisterTransportCallback(this) != 0) {
211    return false;
212  }
213  if (vcm_.RegisterSendStatisticsCallback(this) != 0) {
214    return false;
215  }
216  if (vcm_.RegisterVideoQMCallback(qm_callback_) != 0) {
217    return false;
218  }
219  return true;
220}
221
222ViEEncoder::~ViEEncoder() {
223  if (bitrate_controller_) {
224    bitrate_controller_->RemoveBitrateObserver(bitrate_observer_.get());
225  }
226  module_process_thread_.DeRegisterModule(&vcm_);
227  module_process_thread_.DeRegisterModule(&vpm_);
228  module_process_thread_.DeRegisterModule(default_rtp_rtcp_.get());
229  module_process_thread_.DeRegisterModule(paced_sender_.get());
230  VideoCodingModule::Destroy(&vcm_);
231  VideoProcessingModule::Destroy(&vpm_);
232  delete qm_callback_;
233}
234
235int ViEEncoder::Owner() const {
236  return channel_id_;
237}
238
239void ViEEncoder::SetNetworkTransmissionState(bool is_transmitting) {
240  {
241    CriticalSectionScoped cs(data_cs_.get());
242    network_is_transmitting_ = is_transmitting;
243  }
244  if (is_transmitting) {
245    paced_sender_->Resume();
246  } else {
247    paced_sender_->Pause();
248  }
249}
250
251void ViEEncoder::Pause() {
252  CriticalSectionScoped cs(data_cs_.get());
253  encoder_paused_ = true;
254}
255
256void ViEEncoder::Restart() {
257  CriticalSectionScoped cs(data_cs_.get());
258  encoder_paused_ = false;
259}
260
261uint8_t ViEEncoder::NumberOfCodecs() {
262  return vcm_.NumberOfCodecs();
263}
264
265int32_t ViEEncoder::GetCodec(uint8_t list_index, VideoCodec* video_codec) {
266  if (vcm_.Codec(list_index, video_codec) != 0) {
267    return -1;
268  }
269  return 0;
270}
271
272int32_t ViEEncoder::RegisterExternalEncoder(webrtc::VideoEncoder* encoder,
273                                            uint8_t pl_type,
274                                            bool internal_source) {
275  if (encoder == NULL)
276    return -1;
277
278  if (vcm_.RegisterExternalEncoder(encoder, pl_type, internal_source) !=
279      VCM_OK) {
280    return -1;
281  }
282  return 0;
283}
284
285int32_t ViEEncoder::DeRegisterExternalEncoder(uint8_t pl_type) {
286  webrtc::VideoCodec current_send_codec;
287  if (vcm_.SendCodec(&current_send_codec) == VCM_OK) {
288    uint32_t current_bitrate_bps = 0;
289    if (vcm_.Bitrate(&current_bitrate_bps) != 0) {
290      LOG(LS_WARNING) << "Failed to get the current encoder target bitrate.";
291    }
292    current_send_codec.startBitrate = (current_bitrate_bps + 500) / 1000;
293  }
294
295  if (vcm_.RegisterExternalEncoder(NULL, pl_type) != VCM_OK) {
296    return -1;
297  }
298
299  // If the external encoder is the current send codec, use vcm internal
300  // encoder.
301  if (current_send_codec.plType == pl_type) {
302    uint16_t max_data_payload_length =
303        default_rtp_rtcp_->MaxDataPayloadLength();
304    {
305      CriticalSectionScoped cs(data_cs_.get());
306      send_padding_ = current_send_codec.numberOfSimulcastStreams > 1;
307    }
308    // TODO(mflodman): Unfortunately the VideoCodec that VCM has cached a
309    // raw pointer to an |extra_options| that's long gone.  Clearing it here is
310    // a hack to prevent the following code from crashing.  This should be fixed
311    // for realz.  https://code.google.com/p/chromium/issues/detail?id=348222
312    current_send_codec.extra_options = NULL;
313    if (vcm_.RegisterSendCodec(&current_send_codec, number_of_cores_,
314                               max_data_payload_length) != VCM_OK) {
315      LOG(LS_INFO) << "De-registered the currently used external encoder ("
316                   << static_cast<int>(pl_type) << ") and therefore tried to "
317                   << "register the corresponding internal encoder, but none "
318                   << "was supported.";
319    }
320  }
321  return 0;
322}
323
324int32_t ViEEncoder::SetEncoder(const webrtc::VideoCodec& video_codec) {
325  // Setting target width and height for VPM.
326  if (vpm_.SetTargetResolution(video_codec.width, video_codec.height,
327                               video_codec.maxFramerate) != VPM_OK) {
328    return -1;
329  }
330
331  if (default_rtp_rtcp_->RegisterSendPayload(video_codec) != 0) {
332    return -1;
333  }
334  // Convert from kbps to bps.
335  std::vector<uint32_t> stream_bitrates = AllocateStreamBitrates(
336      video_codec.startBitrate * 1000,
337      video_codec.simulcastStream,
338      video_codec.numberOfSimulcastStreams);
339  default_rtp_rtcp_->SetTargetSendBitrate(stream_bitrates);
340
341  uint16_t max_data_payload_length =
342      default_rtp_rtcp_->MaxDataPayloadLength();
343
344  {
345    CriticalSectionScoped cs(data_cs_.get());
346    send_padding_ = video_codec.numberOfSimulcastStreams > 1;
347  }
348  if (vcm_.RegisterSendCodec(&video_codec, number_of_cores_,
349                             max_data_payload_length) != VCM_OK) {
350    return -1;
351  }
352
353  // Set this module as sending right away, let the slave module in the channel
354  // start and stop sending.
355  if (default_rtp_rtcp_->SetSendingStatus(true) != 0) {
356    return -1;
357  }
358
359  bitrate_controller_->SetBitrateObserver(bitrate_observer_.get(),
360                                          video_codec.startBitrate * 1000,
361                                          video_codec.minBitrate * 1000,
362                                          kTransmissionMaxBitrateMultiplier *
363                                          video_codec.maxBitrate * 1000);
364
365  CriticalSectionScoped crit(data_cs_.get());
366  int pad_up_to_bitrate_kbps = video_codec.startBitrate;
367  if (pad_up_to_bitrate_kbps < min_transmit_bitrate_kbps_)
368    pad_up_to_bitrate_kbps = min_transmit_bitrate_kbps_;
369
370  paced_sender_->UpdateBitrate(
371      PacedSender::kDefaultPaceMultiplier * video_codec.startBitrate,
372      pad_up_to_bitrate_kbps);
373
374  return 0;
375}
376
377int32_t ViEEncoder::GetEncoder(VideoCodec* video_codec) {
378  if (vcm_.SendCodec(video_codec) != 0) {
379    return -1;
380  }
381  return 0;
382}
383
384int32_t ViEEncoder::GetCodecConfigParameters(
385    unsigned char config_parameters[kConfigParameterSize],
386    unsigned char& config_parameters_size) {
387  int32_t num_parameters =
388      vcm_.CodecConfigParameters(config_parameters, kConfigParameterSize);
389  if (num_parameters <= 0) {
390    config_parameters_size = 0;
391    return -1;
392  }
393  config_parameters_size = static_cast<unsigned char>(num_parameters);
394  return 0;
395}
396
397int32_t ViEEncoder::ScaleInputImage(bool enable) {
398  VideoFrameResampling resampling_mode = kFastRescaling;
399  // TODO(mflodman) What?
400  if (enable) {
401    // kInterpolation is currently not supported.
402    LOG_F(LS_ERROR) << "Not supported.";
403    return -1;
404  }
405  vpm_.SetInputFrameResampleMode(resampling_mode);
406
407  return 0;
408}
409
410bool ViEEncoder::TimeToSendPacket(uint32_t ssrc,
411                                  uint16_t sequence_number,
412                                  int64_t capture_time_ms,
413                                  bool retransmission) {
414  return default_rtp_rtcp_->TimeToSendPacket(ssrc, sequence_number,
415                                             capture_time_ms, retransmission);
416}
417
418int ViEEncoder::TimeToSendPadding(int bytes) {
419  bool send_padding;
420  {
421    CriticalSectionScoped cs(data_cs_.get());
422    send_padding =
423        send_padding_ || video_suspended_ || min_transmit_bitrate_kbps_ > 0;
424  }
425  if (send_padding) {
426    return default_rtp_rtcp_->TimeToSendPadding(bytes);
427  }
428  return 0;
429}
430
431bool ViEEncoder::EncoderPaused() const {
432  // Pause video if paused by caller or as long as the network is down or the
433  // pacer queue has grown too large in buffered mode.
434  if (encoder_paused_) {
435    return true;
436  }
437  if (target_delay_ms_ > 0) {
438    // Buffered mode.
439    // TODO(pwestin): Workaround until nack is configured as a time and not
440    // number of packets.
441    return paced_sender_->QueueInMs() >=
442        std::max(static_cast<int>(target_delay_ms_ * kEncoderPausePacerMargin),
443                 kMinPacingDelayMs);
444  }
445  return !network_is_transmitting_;
446}
447
448RtpRtcp* ViEEncoder::SendRtpRtcpModule() {
449  return default_rtp_rtcp_.get();
450}
451
452void ViEEncoder::DeliverFrame(int id,
453                              I420VideoFrame* video_frame,
454                              int num_csrcs,
455                              const uint32_t CSRC[kRtpCsrcSize]) {
456  if (default_rtp_rtcp_->SendingMedia() == false) {
457    // We've paused or we have no channels attached, don't encode.
458    return;
459  }
460  {
461    CriticalSectionScoped cs(data_cs_.get());
462    time_of_last_incoming_frame_ms_ = TickTime::MillisecondTimestamp();
463    if (EncoderPaused()) {
464      if (!encoder_paused_and_dropped_frame_) {
465        TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this);
466      }
467      encoder_paused_and_dropped_frame_ = true;
468      return;
469    }
470    if (encoder_paused_and_dropped_frame_) {
471      TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this);
472    }
473    encoder_paused_and_dropped_frame_ = false;
474  }
475
476  // Convert render time, in ms, to RTP timestamp.
477  const int kMsToRtpTimestamp = 90;
478  const uint32_t time_stamp =
479      kMsToRtpTimestamp *
480      static_cast<uint32_t>(video_frame->render_time_ms());
481
482  TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame->render_time_ms(),
483                          "Encode");
484  video_frame->set_timestamp(time_stamp);
485
486  // Make sure the CSRC list is correct.
487  if (num_csrcs > 0) {
488    uint32_t tempCSRC[kRtpCsrcSize];
489    for (int i = 0; i < num_csrcs; i++) {
490      if (CSRC[i] == 1) {
491        tempCSRC[i] = default_rtp_rtcp_->SSRC();
492      } else {
493        tempCSRC[i] = CSRC[i];
494      }
495    }
496    default_rtp_rtcp_->SetCSRCs(tempCSRC, (uint8_t) num_csrcs);
497  }
498
499  I420VideoFrame* decimated_frame = NULL;
500  // TODO(wuchengli): support texture frames.
501  if (video_frame->native_handle() == NULL) {
502    {
503      CriticalSectionScoped cs(callback_cs_.get());
504      if (effect_filter_) {
505        unsigned int length =
506            CalcBufferSize(kI420, video_frame->width(), video_frame->height());
507        scoped_ptr<uint8_t[]> video_buffer(new uint8_t[length]);
508        ExtractBuffer(*video_frame, length, video_buffer.get());
509        effect_filter_->Transform(length,
510                                  video_buffer.get(),
511                                  video_frame->ntp_time_ms(),
512                                  video_frame->timestamp(),
513                                  video_frame->width(),
514                                  video_frame->height());
515      }
516    }
517
518    // Pass frame via preprocessor.
519    const int ret = vpm_.PreprocessFrame(*video_frame, &decimated_frame);
520    if (ret == 1) {
521      // Drop this frame.
522      return;
523    }
524    if (ret != VPM_OK) {
525      return;
526    }
527  }
528  // If the frame was not resampled or scaled => use original.
529  if (decimated_frame == NULL)  {
530    decimated_frame = video_frame;
531  }
532
533  {
534    CriticalSectionScoped cs(callback_cs_.get());
535    if (pre_encode_callback_)
536      pre_encode_callback_->FrameCallback(decimated_frame);
537  }
538
539  if (video_frame->native_handle() != NULL) {
540    // TODO(wuchengli): add texture support. http://crbug.com/362437
541    return;
542  }
543
544#ifdef VIDEOCODEC_VP8
545  if (vcm_.SendCodec() == webrtc::kVideoCodecVP8) {
546    webrtc::CodecSpecificInfo codec_specific_info;
547    codec_specific_info.codecType = webrtc::kVideoCodecVP8;
548    {
549      CriticalSectionScoped cs(data_cs_.get());
550      codec_specific_info.codecSpecific.VP8.hasReceivedRPSI =
551          has_received_rpsi_;
552      codec_specific_info.codecSpecific.VP8.hasReceivedSLI =
553          has_received_sli_;
554      codec_specific_info.codecSpecific.VP8.pictureIdRPSI =
555          picture_id_rpsi_;
556      codec_specific_info.codecSpecific.VP8.pictureIdSLI  =
557          picture_id_sli_;
558      has_received_sli_ = false;
559      has_received_rpsi_ = false;
560    }
561
562    vcm_.AddVideoFrame(*decimated_frame, vpm_.ContentMetrics(),
563                       &codec_specific_info);
564    return;
565  }
566#endif
567  vcm_.AddVideoFrame(*decimated_frame);
568}
569
570void ViEEncoder::DelayChanged(int id, int frame_delay) {
571  default_rtp_rtcp_->SetCameraDelay(frame_delay);
572}
573
574int ViEEncoder::GetPreferedFrameSettings(int* width,
575                                         int* height,
576                                         int* frame_rate) {
577  webrtc::VideoCodec video_codec;
578  memset(&video_codec, 0, sizeof(video_codec));
579  if (vcm_.SendCodec(&video_codec) != VCM_OK) {
580    return -1;
581  }
582
583  *width = video_codec.width;
584  *height = video_codec.height;
585  *frame_rate = video_codec.maxFramerate;
586  return 0;
587}
588
589int ViEEncoder::SendKeyFrame() {
590  return vcm_.IntraFrameRequest(0);
591}
592
593int32_t ViEEncoder::SendCodecStatistics(
594    uint32_t* num_key_frames, uint32_t* num_delta_frames) {
595  webrtc::VCMFrameCount sent_frames;
596  if (vcm_.SentFrameCount(sent_frames) != VCM_OK) {
597    return -1;
598  }
599  *num_key_frames = sent_frames.numKeyFrames;
600  *num_delta_frames = sent_frames.numDeltaFrames;
601  return 0;
602}
603
604int32_t ViEEncoder::PacerQueuingDelayMs() const {
605  return paced_sender_->QueueInMs();
606}
607
608int ViEEncoder::CodecTargetBitrate(uint32_t* bitrate) const {
609  if (vcm_.Bitrate(bitrate) != 0)
610    return -1;
611  return 0;
612}
613
614int32_t ViEEncoder::UpdateProtectionMethod(bool enable_nack) {
615  bool fec_enabled = false;
616  uint8_t dummy_ptype_red = 0;
617  uint8_t dummy_ptypeFEC = 0;
618
619  // Updated protection method to VCM to get correct packetization sizes.
620  // FEC has larger overhead than NACK -> set FEC if used.
621  int32_t error = default_rtp_rtcp_->GenericFECStatus(fec_enabled,
622                                                      dummy_ptype_red,
623                                                      dummy_ptypeFEC);
624  if (error) {
625    return -1;
626  }
627  if (fec_enabled_ == fec_enabled && nack_enabled_ == enable_nack) {
628    // No change needed, we're already in correct state.
629    return 0;
630  }
631  fec_enabled_ = fec_enabled;
632  nack_enabled_ = enable_nack;
633
634  // Set Video Protection for VCM.
635  if (fec_enabled && nack_enabled_) {
636    vcm_.SetVideoProtection(webrtc::kProtectionNackFEC, true);
637  } else {
638    vcm_.SetVideoProtection(webrtc::kProtectionFEC, fec_enabled_);
639    vcm_.SetVideoProtection(webrtc::kProtectionNackSender, nack_enabled_);
640    vcm_.SetVideoProtection(webrtc::kProtectionNackFEC, false);
641  }
642
643  if (fec_enabled_ || nack_enabled_) {
644    vcm_.RegisterProtectionCallback(this);
645    // The send codec must be registered to set correct MTU.
646    webrtc::VideoCodec codec;
647    if (vcm_.SendCodec(&codec) == 0) {
648      uint16_t max_pay_load = default_rtp_rtcp_->MaxDataPayloadLength();
649      uint32_t current_bitrate_bps = 0;
650      if (vcm_.Bitrate(&current_bitrate_bps) != 0) {
651        LOG_F(LS_WARNING) <<
652            "Failed to get the current encoder target bitrate.";
653      }
654      // Convert to start bitrate in kbps.
655      codec.startBitrate = (current_bitrate_bps + 500) / 1000;
656      if (vcm_.RegisterSendCodec(&codec, number_of_cores_, max_pay_load) != 0) {
657        return -1;
658      }
659    }
660    return 0;
661  } else {
662    // FEC and NACK are disabled.
663    vcm_.RegisterProtectionCallback(NULL);
664  }
665  return 0;
666}
667
668void ViEEncoder::SetSenderBufferingMode(int target_delay_ms) {
669  {
670    CriticalSectionScoped cs(data_cs_.get());
671    target_delay_ms_ = target_delay_ms;
672  }
673  if (target_delay_ms > 0) {
674    // Disable external frame-droppers.
675    vcm_.EnableFrameDropper(false);
676    vpm_.EnableTemporalDecimation(false);
677    // We don't put any limits on the pacer queue when running in buffered mode
678    // since the encoder will be paused if the queue grow too large.
679    paced_sender_->set_max_queue_length_ms(-1);
680  } else {
681    // Real-time mode - enable frame droppers.
682    vpm_.EnableTemporalDecimation(true);
683    vcm_.EnableFrameDropper(true);
684    paced_sender_->set_max_queue_length_ms(
685        PacedSender::kDefaultMaxQueueLengthMs);
686  }
687}
688
689int32_t ViEEncoder::SendData(
690    const FrameType frame_type,
691    const uint8_t payload_type,
692    const uint32_t time_stamp,
693    int64_t capture_time_ms,
694    const uint8_t* payload_data,
695    const uint32_t payload_size,
696    const webrtc::RTPFragmentationHeader& fragmentation_header,
697    const RTPVideoHeader* rtp_video_hdr) {
698  // New encoded data, hand over to the rtp module.
699  return default_rtp_rtcp_->SendOutgoingData(frame_type,
700                                             payload_type,
701                                             time_stamp,
702                                             capture_time_ms,
703                                             payload_data,
704                                             payload_size,
705                                             &fragmentation_header,
706                                             rtp_video_hdr);
707}
708
709int32_t ViEEncoder::ProtectionRequest(
710    const FecProtectionParams* delta_fec_params,
711    const FecProtectionParams* key_fec_params,
712    uint32_t* sent_video_rate_bps,
713    uint32_t* sent_nack_rate_bps,
714    uint32_t* sent_fec_rate_bps) {
715  default_rtp_rtcp_->SetFecParameters(delta_fec_params, key_fec_params);
716  default_rtp_rtcp_->BitrateSent(NULL, sent_video_rate_bps, sent_fec_rate_bps,
717                                sent_nack_rate_bps);
718  return 0;
719}
720
721int32_t ViEEncoder::SendStatistics(const uint32_t bit_rate,
722                                   const uint32_t frame_rate) {
723  CriticalSectionScoped cs(callback_cs_.get());
724  if (codec_observer_) {
725    codec_observer_->OutgoingRate(channel_id_, frame_rate, bit_rate);
726  }
727  return 0;
728}
729
730int32_t ViEEncoder::RegisterCodecObserver(ViEEncoderObserver* observer) {
731  CriticalSectionScoped cs(callback_cs_.get());
732  if (observer && codec_observer_) {
733    LOG_F(LS_ERROR) << "Observer already set.";
734    return -1;
735  }
736  codec_observer_ = observer;
737  return 0;
738}
739
740void ViEEncoder::OnReceivedSLI(uint32_t /*ssrc*/,
741                               uint8_t picture_id) {
742  CriticalSectionScoped cs(data_cs_.get());
743  picture_id_sli_ = picture_id;
744  has_received_sli_ = true;
745}
746
747void ViEEncoder::OnReceivedRPSI(uint32_t /*ssrc*/,
748                                uint64_t picture_id) {
749  CriticalSectionScoped cs(data_cs_.get());
750  picture_id_rpsi_ = picture_id;
751  has_received_rpsi_ = true;
752}
753
754void ViEEncoder::OnReceivedIntraFrameRequest(uint32_t ssrc) {
755  // Key frame request from remote side, signal to VCM.
756  TRACE_EVENT0("webrtc", "OnKeyFrameRequest");
757
758  int idx = 0;
759  {
760    CriticalSectionScoped cs(data_cs_.get());
761    std::map<unsigned int, int>::iterator stream_it = ssrc_streams_.find(ssrc);
762    if (stream_it == ssrc_streams_.end()) {
763      LOG_F(LS_WARNING) << "ssrc not found: " << ssrc << ", map size "
764                        << ssrc_streams_.size();
765      return;
766    }
767    std::map<unsigned int, int64_t>::iterator time_it =
768        time_last_intra_request_ms_.find(ssrc);
769    if (time_it == time_last_intra_request_ms_.end()) {
770      time_last_intra_request_ms_[ssrc] = 0;
771    }
772
773    int64_t now = TickTime::MillisecondTimestamp();
774    if (time_last_intra_request_ms_[ssrc] + kViEMinKeyRequestIntervalMs > now) {
775      return;
776    }
777    time_last_intra_request_ms_[ssrc] = now;
778    idx = stream_it->second;
779  }
780  // Release the critsect before triggering key frame.
781  vcm_.IntraFrameRequest(idx);
782}
783
784void ViEEncoder::OnLocalSsrcChanged(uint32_t old_ssrc, uint32_t new_ssrc) {
785  CriticalSectionScoped cs(data_cs_.get());
786  std::map<unsigned int, int>::iterator it = ssrc_streams_.find(old_ssrc);
787  if (it == ssrc_streams_.end()) {
788    return;
789  }
790
791  ssrc_streams_[new_ssrc] = it->second;
792  ssrc_streams_.erase(it);
793
794  std::map<unsigned int, int64_t>::iterator time_it =
795      time_last_intra_request_ms_.find(old_ssrc);
796  int64_t last_intra_request_ms = 0;
797  if (time_it != time_last_intra_request_ms_.end()) {
798    last_intra_request_ms = time_it->second;
799    time_last_intra_request_ms_.erase(time_it);
800  }
801  time_last_intra_request_ms_[new_ssrc] = last_intra_request_ms;
802}
803
804bool ViEEncoder::SetSsrcs(const std::list<unsigned int>& ssrcs) {
805  VideoCodec codec;
806  if (vcm_.SendCodec(&codec) != 0)
807    return false;
808
809  if (codec.numberOfSimulcastStreams > 0 &&
810      ssrcs.size() != codec.numberOfSimulcastStreams) {
811    return false;
812  }
813
814  CriticalSectionScoped cs(data_cs_.get());
815  ssrc_streams_.clear();
816  time_last_intra_request_ms_.clear();
817  int idx = 0;
818  for (std::list<unsigned int>::const_iterator it = ssrcs.begin();
819       it != ssrcs.end(); ++it, ++idx) {
820    unsigned int ssrc = *it;
821    ssrc_streams_[ssrc] = idx;
822  }
823  return true;
824}
825
826void ViEEncoder::SetMinTransmitBitrate(int min_transmit_bitrate_kbps) {
827  assert(min_transmit_bitrate_kbps >= 0);
828  CriticalSectionScoped crit(data_cs_.get());
829  min_transmit_bitrate_kbps_ = min_transmit_bitrate_kbps;
830}
831
832// Called from ViEBitrateObserver.
833void ViEEncoder::OnNetworkChanged(const uint32_t bitrate_bps,
834                                  const uint8_t fraction_lost,
835                                  const uint32_t round_trip_time_ms) {
836  LOG(LS_VERBOSE) << "OnNetworkChanged, bitrate" << bitrate_bps
837                  << " packet loss " << fraction_lost
838                  << " rtt " << round_trip_time_ms;
839  vcm_.SetChannelParameters(bitrate_bps, fraction_lost, round_trip_time_ms);
840  bool video_is_suspended = vcm_.VideoSuspended();
841  int bitrate_kbps = bitrate_bps / 1000;
842  VideoCodec send_codec;
843  if (vcm_.SendCodec(&send_codec) != 0) {
844    return;
845  }
846  SimulcastStream* stream_configs = send_codec.simulcastStream;
847  // Allocate the bandwidth between the streams.
848  std::vector<uint32_t> stream_bitrates = AllocateStreamBitrates(
849      bitrate_bps,
850      stream_configs,
851      send_codec.numberOfSimulcastStreams);
852  // Find the max amount of padding we can allow ourselves to send at this
853  // point, based on which streams are currently active and what our current
854  // available bandwidth is.
855  int pad_up_to_bitrate_kbps = 0;
856  if (send_codec.numberOfSimulcastStreams == 0) {
857    pad_up_to_bitrate_kbps = send_codec.minBitrate;
858  } else {
859    pad_up_to_bitrate_kbps =
860        stream_configs[send_codec.numberOfSimulcastStreams - 1].minBitrate;
861    for (int i = 0; i < send_codec.numberOfSimulcastStreams - 1; ++i) {
862      pad_up_to_bitrate_kbps += stream_configs[i].targetBitrate;
863    }
864  }
865
866  // Disable padding if only sending one stream and video isn't suspended and
867  // min-transmit bitrate isn't used (applied later).
868  if (!video_is_suspended && send_codec.numberOfSimulcastStreams <= 1)
869    pad_up_to_bitrate_kbps = 0;
870
871  {
872    CriticalSectionScoped cs(data_cs_.get());
873    // The amount of padding should decay to zero if no frames are being
874    // captured unless a min-transmit bitrate is used.
875    int64_t now_ms = TickTime::MillisecondTimestamp();
876    if (now_ms - time_of_last_incoming_frame_ms_ > kStopPaddingThresholdMs)
877      pad_up_to_bitrate_kbps = 0;
878
879    // Pad up to min bitrate.
880    if (pad_up_to_bitrate_kbps < min_transmit_bitrate_kbps_)
881      pad_up_to_bitrate_kbps = min_transmit_bitrate_kbps_;
882
883    // Padding may never exceed bitrate estimate.
884    if (pad_up_to_bitrate_kbps > bitrate_kbps)
885      pad_up_to_bitrate_kbps = bitrate_kbps;
886
887    paced_sender_->UpdateBitrate(
888        PacedSender::kDefaultPaceMultiplier * bitrate_kbps,
889        pad_up_to_bitrate_kbps);
890    default_rtp_rtcp_->SetTargetSendBitrate(stream_bitrates);
891    if (video_suspended_ == video_is_suspended)
892      return;
893    video_suspended_ = video_is_suspended;
894  }
895
896  // Video suspend-state changed, inform codec observer.
897  CriticalSectionScoped crit(callback_cs_.get());
898  if (codec_observer_) {
899    LOG(LS_INFO) << "Video suspended " << video_is_suspended
900                 << " for channel " << channel_id_;
901    codec_observer_->SuspendChange(channel_id_, video_is_suspended);
902  }
903}
904
905PacedSender* ViEEncoder::GetPacedSender() {
906  return paced_sender_.get();
907}
908
909int32_t ViEEncoder::RegisterEffectFilter(ViEEffectFilter* effect_filter) {
910  CriticalSectionScoped cs(callback_cs_.get());
911  if (effect_filter != NULL && effect_filter_ != NULL) {
912    LOG_F(LS_ERROR) << "Filter already set.";
913    return -1;
914  }
915  effect_filter_ = effect_filter;
916  return 0;
917}
918
919int ViEEncoder::StartDebugRecording(const char* fileNameUTF8) {
920  return vcm_.StartDebugRecording(fileNameUTF8);
921}
922
923int ViEEncoder::StopDebugRecording() {
924  return vcm_.StopDebugRecording();
925}
926
927void ViEEncoder::SuspendBelowMinBitrate() {
928  vcm_.SuspendBelowMinBitrate();
929  bitrate_controller_->EnforceMinBitrate(false);
930}
931
932void ViEEncoder::RegisterPreEncodeCallback(
933    I420FrameCallback* pre_encode_callback) {
934  CriticalSectionScoped cs(callback_cs_.get());
935  pre_encode_callback_ = pre_encode_callback;
936}
937
938void ViEEncoder::DeRegisterPreEncodeCallback() {
939  CriticalSectionScoped cs(callback_cs_.get());
940  pre_encode_callback_ = NULL;
941}
942
943void ViEEncoder::RegisterPostEncodeImageCallback(
944      EncodedImageCallback* post_encode_callback) {
945  vcm_.RegisterPostEncodeImageCallback(post_encode_callback);
946}
947
948void ViEEncoder::DeRegisterPostEncodeImageCallback() {
949  vcm_.RegisterPostEncodeImageCallback(NULL);
950}
951
952QMVideoSettingsCallback::QMVideoSettingsCallback(VideoProcessingModule* vpm)
953    : vpm_(vpm) {
954}
955
956QMVideoSettingsCallback::~QMVideoSettingsCallback() {
957}
958
959int32_t QMVideoSettingsCallback::SetVideoQMSettings(
960    const uint32_t frame_rate,
961    const uint32_t width,
962    const uint32_t height) {
963  return vpm_->SetTargetResolution(width, height, frame_rate);
964}
965
966}  // namespace webrtc
967