1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4//
5// TODO (pwestin): add a link to the design document describing the generic
6// protocol and the VP8 specific details.
7#include "media/cast/video_sender/codecs/vp8/vp8_encoder.h"
8
9#include <vector>
10
11#include "base/logging.h"
12#include "media/base/video_frame.h"
13#include "media/cast/cast_defines.h"
14#include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
15
16namespace media {
17namespace cast {
18
19static const uint32 kMinIntra = 300;
20
21Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config,
22                       uint8 max_unacked_frames)
23    : cast_config_(video_config),
24      use_multiple_video_buffers_(
25          cast_config_.max_number_of_video_buffers_used ==
26          kNumberOfVp8VideoBuffers),
27      max_number_of_repeated_buffers_in_a_row_(
28          (max_unacked_frames > kNumberOfVp8VideoBuffers) ?
29          ((max_unacked_frames - 1) / kNumberOfVp8VideoBuffers) : 0),
30      config_(new vpx_codec_enc_cfg_t()),
31      encoder_(new vpx_codec_ctx_t()),
32      // Creating a wrapper to the image - setting image data to NULL. Actual
33      // pointer will be set during encode. Setting align to 1, as it is
34      // meaningless (actual memory is not allocated).
35      raw_image_(vpx_img_wrap(NULL, IMG_FMT_I420, video_config.width,
36                              video_config.height, 1, NULL)),
37      key_frame_requested_(true),
38      timestamp_(0),
39      last_encoded_frame_id_(kStartFrameId),
40      number_of_repeated_buffers_(0) {
41  // TODO(pwestin): we need to figure out how to synchronize the acking with the
42  // internal state of the encoder, ideally the encoder will tell if we can
43  // send another frame.
44  DCHECK(!use_multiple_video_buffers_ ||
45         max_number_of_repeated_buffers_in_a_row_ == 0) <<  "Invalid config";
46
47  // VP8 have 3 buffers available for prediction, with
48  // max_number_of_video_buffers_used set to 1 we maximize the coding efficiency
49  // however in this mode we can not skip frames in the receiver to catch up
50  // after a temporary network outage; with max_number_of_video_buffers_used
51  // set to 3 we allow 2 frames to be skipped by the receiver without error
52  // propagation.
53  DCHECK(cast_config_.max_number_of_video_buffers_used == 1 ||
54         cast_config_.max_number_of_video_buffers_used ==
55             kNumberOfVp8VideoBuffers) << "Invalid argument";
56
57  for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
58    acked_frame_buffers_[i] = true;
59    used_buffers_frame_id_[i] = kStartFrameId;
60  }
61  InitEncode(video_config.number_of_cores);
62}
63
64Vp8Encoder::~Vp8Encoder() {
65  vpx_codec_destroy(encoder_);
66  vpx_img_free(raw_image_);
67}
68
69void Vp8Encoder::InitEncode(int number_of_cores) {
70  // Populate encoder configuration with default values.
71  if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), config_.get(), 0)) {
72    DCHECK(false) << "Invalid return value";
73  }
74  config_->g_w = cast_config_.width;
75  config_->g_h = cast_config_.height;
76  config_->rc_target_bitrate = cast_config_.start_bitrate / 1000;  // In kbit/s.
77
78  // Setting the codec time base.
79  config_->g_timebase.num = 1;
80  config_->g_timebase.den = kVideoFrequency;
81  config_->g_lag_in_frames = 0;
82  config_->kf_mode = VPX_KF_DISABLED;
83  if (use_multiple_video_buffers_) {
84    // We must enable error resilience when we use multiple buffers, due to
85    // codec requirements.
86    config_->g_error_resilient = 1;
87  }
88
89  if (cast_config_.width * cast_config_.height > 640 * 480
90      && number_of_cores >= 2) {
91    config_->g_threads = 2;  // 2 threads for qHD/HD.
92  } else {
93    config_->g_threads = 1;  // 1 thread for VGA or less.
94  }
95
96  // Rate control settings.
97  // TODO(pwestin): revisit these constants. Currently identical to webrtc.
98  config_->rc_dropframe_thresh = 30;
99  config_->rc_end_usage = VPX_CBR;
100  config_->g_pass = VPX_RC_ONE_PASS;
101  config_->rc_resize_allowed = 0;
102  config_->rc_min_quantizer = cast_config_.min_qp;
103  config_->rc_max_quantizer = cast_config_.max_qp;
104  config_->rc_undershoot_pct = 100;
105  config_->rc_overshoot_pct = 15;
106  config_->rc_buf_initial_sz = 500;
107  config_->rc_buf_optimal_sz = 600;
108  config_->rc_buf_sz = 1000;
109
110  // set the maximum target size of any key-frame.
111  uint32 rc_max_intra_target = MaxIntraTarget(config_->rc_buf_optimal_sz);
112  vpx_codec_flags_t flags = 0;
113  // TODO(mikhal): Tune settings.
114  if (vpx_codec_enc_init(encoder_, vpx_codec_vp8_cx(), config_.get(), flags)) {
115    DCHECK(false) << "Invalid return value";
116  }
117  vpx_codec_control(encoder_, VP8E_SET_STATIC_THRESHOLD, 1);
118  vpx_codec_control(encoder_, VP8E_SET_NOISE_SENSITIVITY, 0);
119  vpx_codec_control(encoder_, VP8E_SET_CPUUSED, -6);
120  vpx_codec_control(encoder_, VP8E_SET_MAX_INTRA_BITRATE_PCT,
121                    rc_max_intra_target);
122}
123
124bool Vp8Encoder::Encode(const scoped_refptr<media::VideoFrame>& video_frame,
125                        EncodedVideoFrame* encoded_image) {
126  // Image in vpx_image_t format.
127  // Input image is const. VP8's raw image is not defined as const.
128  raw_image_->planes[PLANE_Y] =
129      const_cast<uint8*>(video_frame->data(VideoFrame::kYPlane));
130  raw_image_->planes[PLANE_U] =
131      const_cast<uint8*>(video_frame->data(VideoFrame::kUPlane));
132  raw_image_->planes[PLANE_V] =
133      const_cast<uint8*>(video_frame->data(VideoFrame::kVPlane));
134
135  raw_image_->stride[VPX_PLANE_Y] = video_frame->stride(VideoFrame::kYPlane);
136  raw_image_->stride[VPX_PLANE_U] = video_frame->stride(VideoFrame::kUPlane);
137  raw_image_->stride[VPX_PLANE_V] = video_frame->stride(VideoFrame::kVPlane);
138
139  uint8 latest_frame_id_to_reference;
140  Vp8Buffers buffer_to_update;
141  vpx_codec_flags_t flags = 0;
142  if (key_frame_requested_) {
143    flags = VPX_EFLAG_FORCE_KF;
144    // Self reference.
145    latest_frame_id_to_reference =
146        static_cast<uint8>(last_encoded_frame_id_ + 1);
147    // We can pick any buffer as buffer_to_update since we update
148    // them all.
149    buffer_to_update = kLastBuffer;
150  } else {
151    // Reference all acked frames (buffers).
152    latest_frame_id_to_reference = GetLatestFrameIdToReference();
153    GetCodecReferenceFlags(&flags);
154    buffer_to_update = GetNextBufferToUpdate();
155    GetCodecUpdateFlags(buffer_to_update, &flags);
156  }
157
158  // Note: The duration does not reflect the real time between frames. This is
159  // done to keep the encoder happy.
160  uint32 duration = kVideoFrequency / cast_config_.max_frame_rate;
161  if (vpx_codec_encode(encoder_, raw_image_, timestamp_, duration, flags,
162                       VPX_DL_REALTIME)) {
163    return false;
164  }
165  timestamp_ += duration;
166
167  // Get encoded frame.
168  const vpx_codec_cx_pkt_t *pkt = NULL;
169  vpx_codec_iter_t iter = NULL;
170  size_t total_size = 0;
171  while ((pkt = vpx_codec_get_cx_data(encoder_, &iter)) != NULL) {
172    if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) {
173      total_size += pkt->data.frame.sz;
174      encoded_image->data.reserve(total_size);
175      encoded_image->data.insert(
176          encoded_image->data.end(),
177          static_cast<const uint8*>(pkt->data.frame.buf),
178          static_cast<const uint8*>(pkt->data.frame.buf) +
179              pkt->data.frame.sz);
180      if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) {
181        encoded_image->key_frame = true;
182      } else {
183        encoded_image->key_frame = false;
184      }
185    }
186  }
187  // Don't update frame_id for zero size frames.
188  if (total_size == 0) return true;
189
190  // Populate the encoded frame.
191  encoded_image->codec = kVp8;
192  encoded_image->last_referenced_frame_id = latest_frame_id_to_reference;
193  encoded_image->frame_id = ++last_encoded_frame_id_;
194
195  VLOG(1) << "VP8 encoded frame:" << static_cast<int>(encoded_image->frame_id)
196          << " sized:" << total_size;
197
198  if (encoded_image->key_frame) {
199    key_frame_requested_ = false;
200
201    for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
202      used_buffers_frame_id_[i] = encoded_image->frame_id;
203    }
204    // We can pick any buffer as last_used_vp8_buffer_ since we update
205    // them all.
206    last_used_vp8_buffer_ = buffer_to_update;
207  } else {
208    if (buffer_to_update != kNoBuffer) {
209      acked_frame_buffers_[buffer_to_update] = false;
210      used_buffers_frame_id_[buffer_to_update] = encoded_image->frame_id;
211      last_used_vp8_buffer_ = buffer_to_update;
212    }
213  }
214  return true;
215}
216
217void Vp8Encoder::GetCodecReferenceFlags(vpx_codec_flags_t* flags) {
218  if (!use_multiple_video_buffers_) return;
219
220  // We need to reference something.
221  DCHECK(acked_frame_buffers_[kAltRefBuffer] ||
222         acked_frame_buffers_[kGoldenBuffer] ||
223         acked_frame_buffers_[kLastBuffer]) << "Invalid state";
224
225  if (!acked_frame_buffers_[kAltRefBuffer]) {
226    *flags |= VP8_EFLAG_NO_REF_ARF;
227  }
228  if (!acked_frame_buffers_[kGoldenBuffer]) {
229    *flags |= VP8_EFLAG_NO_REF_GF;
230  }
231  if (!acked_frame_buffers_[kLastBuffer]) {
232    *flags |= VP8_EFLAG_NO_REF_LAST;
233  }
234}
235
236uint32 Vp8Encoder::GetLatestFrameIdToReference() {
237  if (!use_multiple_video_buffers_) return last_encoded_frame_id_;
238
239  int latest_frame_id_to_reference = -1;
240  if (acked_frame_buffers_[kAltRefBuffer]) {
241    latest_frame_id_to_reference = used_buffers_frame_id_[kAltRefBuffer];
242  }
243  if (acked_frame_buffers_[kGoldenBuffer]) {
244    if (latest_frame_id_to_reference == -1) {
245      latest_frame_id_to_reference = used_buffers_frame_id_[kGoldenBuffer];
246    } else {
247      if (IsNewerFrameId(used_buffers_frame_id_[kGoldenBuffer],
248                         latest_frame_id_to_reference)) {
249        latest_frame_id_to_reference = used_buffers_frame_id_[kGoldenBuffer];
250      }
251    }
252  }
253  if (acked_frame_buffers_[kLastBuffer]) {
254    if (latest_frame_id_to_reference == -1) {
255      latest_frame_id_to_reference = used_buffers_frame_id_[kLastBuffer];
256    } else {
257      if (IsNewerFrameId(used_buffers_frame_id_[kLastBuffer],
258                         latest_frame_id_to_reference)) {
259        latest_frame_id_to_reference = used_buffers_frame_id_[kLastBuffer];
260      }
261    }
262  }
263  DCHECK(latest_frame_id_to_reference != -1) << "Invalid state";
264  return static_cast<uint32>(latest_frame_id_to_reference);
265}
266
267Vp8Encoder::Vp8Buffers Vp8Encoder::GetNextBufferToUpdate() {
268  // Update at most one buffer, except for key-frames.
269
270  Vp8Buffers buffer_to_update;
271  if (number_of_repeated_buffers_ < max_number_of_repeated_buffers_in_a_row_) {
272    // TODO(pwestin): experiment with this. The issue with only this change is
273    // that we can end up with only 4 frames in flight when we expect 6.
274    // buffer_to_update = last_used_vp8_buffer_;
275    buffer_to_update = kNoBuffer;
276    ++number_of_repeated_buffers_;
277  } else {
278    number_of_repeated_buffers_ = 0;
279    switch (last_used_vp8_buffer_) {
280      case kAltRefBuffer:
281        buffer_to_update = kLastBuffer;
282        VLOG(1) << "VP8 update last buffer";
283        break;
284      case kLastBuffer:
285        buffer_to_update = kGoldenBuffer;
286        VLOG(1) << "VP8 update golden buffer";
287        break;
288      case kGoldenBuffer:
289        buffer_to_update = kAltRefBuffer;
290        VLOG(1) << "VP8 update alt-ref buffer";
291        break;
292      case kNoBuffer:
293        DCHECK(false) << "Invalid state";
294        break;
295    }
296  }
297  return buffer_to_update;
298}
299
300void Vp8Encoder::GetCodecUpdateFlags(Vp8Buffers buffer_to_update,
301                                     vpx_codec_flags_t* flags) {
302  if (!use_multiple_video_buffers_) return;
303
304  // Update at most one buffer, except for key-frames.
305  switch (buffer_to_update) {
306    case kAltRefBuffer:
307      *flags |= VP8_EFLAG_NO_UPD_GF;
308      *flags |= VP8_EFLAG_NO_UPD_LAST;
309      break;
310    case kLastBuffer:
311      *flags |= VP8_EFLAG_NO_UPD_GF;
312      *flags |= VP8_EFLAG_NO_UPD_ARF;
313      break;
314    case kGoldenBuffer:
315      *flags |= VP8_EFLAG_NO_UPD_ARF;
316      *flags |= VP8_EFLAG_NO_UPD_LAST;
317      break;
318    case kNoBuffer:
319      *flags |= VP8_EFLAG_NO_UPD_ARF;
320      *flags |= VP8_EFLAG_NO_UPD_GF;
321      *flags |= VP8_EFLAG_NO_UPD_LAST;
322      *flags |= VP8_EFLAG_NO_UPD_ENTROPY;
323      break;
324  }
325}
326
327void Vp8Encoder::UpdateRates(uint32 new_bitrate) {
328  uint32 new_bitrate_kbit = new_bitrate / 1000;
329  if (config_->rc_target_bitrate == new_bitrate_kbit) return;
330
331  config_->rc_target_bitrate = new_bitrate_kbit;
332
333  // Update encoder context.
334  if (vpx_codec_enc_config_set(encoder_, config_.get())) {
335    DCHECK(false) << "Invalid return value";
336  }
337}
338
339void Vp8Encoder::LatestFrameIdToReference(uint32 frame_id) {
340  if (!use_multiple_video_buffers_) return;
341
342  VLOG(1) << "VP8 ok to reference frame:" << static_cast<int>(frame_id);
343  for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
344    if (frame_id == used_buffers_frame_id_[i]) {
345      acked_frame_buffers_[i] = true;
346    }
347  }
348}
349
350void Vp8Encoder::GenerateKeyFrame() {
351  key_frame_requested_ = true;
352}
353
354// Calculate the max size of the key frame relative to a normal delta frame.
355uint32 Vp8Encoder::MaxIntraTarget(uint32 optimal_buffer_size_ms) const {
356  // Set max to the optimal buffer level (normalized by target BR),
357  // and scaled by a scale_parameter.
358  // Max target size = scalePar * optimalBufferSize * targetBR[Kbps].
359  // This values is presented in percentage of perFrameBw:
360  // perFrameBw = targetBR[Kbps] * 1000 / frameRate.
361  // The target in % is as follows:
362
363  float scale_parameter = 0.5;
364  uint32 target_pct = optimal_buffer_size_ms * scale_parameter *
365      cast_config_.max_frame_rate / 10;
366
367  // Don't go below 3 times the per frame bandwidth.
368  return std::max(target_pct, kMinIntra);
369}
370
371}  // namespace cast
372}  // namespace media
373