1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/modules/video_coding/codecs/test/videoprocessor.h"
12
13#include <assert.h>
14#include <string.h>
15
16#include <limits>
17#include <vector>
18
19#include "webrtc/system_wrappers/interface/cpu_info.h"
20
21namespace webrtc {
22namespace test {
23
24TestConfig::TestConfig()
25    : name(""),
26      description(""),
27      test_number(0),
28      input_filename(""),
29      output_filename(""),
30      output_dir("out"),
31      networking_config(),
32      exclude_frame_types(kExcludeOnlyFirstKeyFrame),
33      frame_length_in_bytes(-1),
34      use_single_core(false),
35      keyframe_interval(0),
36      codec_settings(NULL),
37      verbose(true) {}
38
39TestConfig::~TestConfig() {}
40
41VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder,
42                                       webrtc::VideoDecoder* decoder,
43                                       FrameReader* frame_reader,
44                                       FrameWriter* frame_writer,
45                                       PacketManipulator* packet_manipulator,
46                                       const TestConfig& config,
47                                       Stats* stats)
48    : encoder_(encoder),
49      decoder_(decoder),
50      frame_reader_(frame_reader),
51      frame_writer_(frame_writer),
52      packet_manipulator_(packet_manipulator),
53      config_(config),
54      stats_(stats),
55      encode_callback_(NULL),
56      decode_callback_(NULL),
57      source_buffer_(NULL),
58      first_key_frame_has_been_excluded_(false),
59      last_frame_missing_(false),
60      initialized_(false),
61      encoded_frame_size_(0),
62      prev_time_stamp_(0),
63      num_dropped_frames_(0),
64      num_spatial_resizes_(0),
65      last_encoder_frame_width_(0),
66      last_encoder_frame_height_(0),
67      scaler_() {
68  assert(encoder);
69  assert(decoder);
70  assert(frame_reader);
71  assert(frame_writer);
72  assert(packet_manipulator);
73  assert(stats);
74}
75
76bool VideoProcessorImpl::Init() {
77  // Calculate a factor used for bit rate calculations:
78  bit_rate_factor_ = config_.codec_settings->maxFramerate * 0.001 * 8;  // bits
79
80  // Initialize data structures used by the encoder/decoder APIs
81  size_t frame_length_in_bytes = frame_reader_->FrameLength();
82  source_buffer_ = new uint8_t[frame_length_in_bytes];
83  last_successful_frame_buffer_ = new uint8_t[frame_length_in_bytes];
84  // Set fixed properties common for all frames.
85  // To keep track of spatial resize actions by encoder.
86  last_encoder_frame_width_ = config_.codec_settings->width;
87  last_encoder_frame_height_ = config_.codec_settings->height;
88
89  // Setup required callbacks for the encoder/decoder:
90  encode_callback_ = new VideoProcessorEncodeCompleteCallback(this);
91  decode_callback_ = new VideoProcessorDecodeCompleteCallback(this);
92  int32_t register_result =
93      encoder_->RegisterEncodeCompleteCallback(encode_callback_);
94  if (register_result != WEBRTC_VIDEO_CODEC_OK) {
95    fprintf(stderr, "Failed to register encode complete callback, return code: "
96        "%d\n", register_result);
97    return false;
98  }
99  register_result = decoder_->RegisterDecodeCompleteCallback(decode_callback_);
100  if (register_result != WEBRTC_VIDEO_CODEC_OK) {
101    fprintf(stderr, "Failed to register decode complete callback, return code: "
102            "%d\n", register_result);
103    return false;
104  }
105  // Init the encoder and decoder
106  uint32_t nbr_of_cores = 1;
107  if (!config_.use_single_core) {
108    nbr_of_cores = CpuInfo::DetectNumberOfCores();
109  }
110  int32_t init_result =
111      encoder_->InitEncode(config_.codec_settings, nbr_of_cores,
112                           config_.networking_config.max_payload_size_in_bytes);
113  if (init_result != WEBRTC_VIDEO_CODEC_OK) {
114    fprintf(stderr, "Failed to initialize VideoEncoder, return code: %d\n",
115            init_result);
116    return false;
117  }
118  init_result = decoder_->InitDecode(config_.codec_settings, nbr_of_cores);
119  if (init_result != WEBRTC_VIDEO_CODEC_OK) {
120    fprintf(stderr, "Failed to initialize VideoDecoder, return code: %d\n",
121            init_result);
122    return false;
123  }
124
125  if (config_.verbose) {
126    printf("Video Processor:\n");
127    printf("  #CPU cores used  : %d\n", nbr_of_cores);
128    printf("  Total # of frames: %d\n", frame_reader_->NumberOfFrames());
129    printf("  Codec settings:\n");
130    printf("    Start bitrate  : %d kbps\n",
131           config_.codec_settings->startBitrate);
132    printf("    Width          : %d\n", config_.codec_settings->width);
133    printf("    Height         : %d\n", config_.codec_settings->height);
134  }
135  initialized_ = true;
136  return true;
137}
138
139VideoProcessorImpl::~VideoProcessorImpl() {
140  delete[] source_buffer_;
141  delete[] last_successful_frame_buffer_;
142  encoder_->RegisterEncodeCompleteCallback(NULL);
143  delete encode_callback_;
144  decoder_->RegisterDecodeCompleteCallback(NULL);
145  delete decode_callback_;
146}
147
148
149void VideoProcessorImpl::SetRates(int bit_rate, int frame_rate) {
150  int set_rates_result = encoder_->SetRates(bit_rate, frame_rate);
151  assert(set_rates_result >= 0);
152  if (set_rates_result < 0) {
153    fprintf(stderr, "Failed to update encoder with new rate %d, "
154            "return code: %d\n", bit_rate, set_rates_result);
155  }
156  num_dropped_frames_ = 0;
157  num_spatial_resizes_ = 0;
158}
159
160int VideoProcessorImpl::EncodedFrameSize() {
161  return encoded_frame_size_;
162}
163
164int VideoProcessorImpl::NumberDroppedFrames() {
165  return num_dropped_frames_;
166}
167
168int VideoProcessorImpl::NumberSpatialResizes() {
169  return num_spatial_resizes_;
170}
171
172bool VideoProcessorImpl::ProcessFrame(int frame_number) {
173  assert(frame_number >=0);
174  if (!initialized_) {
175    fprintf(stderr, "Attempting to use uninitialized VideoProcessor!\n");
176    return false;
177  }
178  // |prev_time_stamp_| is used for getting number of dropped frames.
179  if (frame_number == 0) {
180    prev_time_stamp_ = -1;
181  }
182  if (frame_reader_->ReadFrame(source_buffer_)) {
183    // Copy the source frame to the newly read frame data.
184    int size_y = config_.codec_settings->width * config_.codec_settings->height;
185    int half_width = (config_.codec_settings->width + 1) / 2;
186    int half_height = (config_.codec_settings->height + 1) / 2;
187    int size_uv = half_width * half_height;
188    source_frame_.CreateFrame(size_y, source_buffer_,
189                              size_uv, source_buffer_ + size_y,
190                              size_uv, source_buffer_ + size_y + size_uv,
191                              config_.codec_settings->width,
192                              config_.codec_settings->height,
193                              config_.codec_settings->width,
194                              half_width, half_width);
195
196    // Ensure we have a new statistics data object we can fill:
197    FrameStatistic& stat = stats_->NewFrame(frame_number);
198
199    encode_start_ = TickTime::Now();
200    // Use the frame number as "timestamp" to identify frames
201    source_frame_.set_timestamp(frame_number);
202
203    // Decide if we're going to force a keyframe:
204    std::vector<VideoFrameType> frame_types(1, kDeltaFrame);
205    if (config_.keyframe_interval > 0 &&
206        frame_number % config_.keyframe_interval == 0) {
207      frame_types[0] = kKeyFrame;
208    }
209
210    // For dropped frames, we regard them as zero size encoded frames.
211    encoded_frame_size_ = 0;
212
213    int32_t encode_result = encoder_->Encode(source_frame_, NULL, &frame_types);
214
215    if (encode_result != WEBRTC_VIDEO_CODEC_OK) {
216      fprintf(stderr, "Failed to encode frame %d, return code: %d\n",
217              frame_number, encode_result);
218    }
219    stat.encode_return_code = encode_result;
220    return true;
221  } else {
222    return false;  // we've reached the last frame
223  }
224}
225
226void VideoProcessorImpl::FrameEncoded(EncodedImage* encoded_image) {
227  // Timestamp is frame number, so this gives us #dropped frames.
228  int num_dropped_from_prev_encode =  encoded_image->_timeStamp -
229      prev_time_stamp_ - 1;
230  num_dropped_frames_ +=  num_dropped_from_prev_encode;
231  prev_time_stamp_ =  encoded_image->_timeStamp;
232  if (num_dropped_from_prev_encode > 0) {
233    // For dropped frames, we write out the last decoded frame to avoid getting
234    // out of sync for the computation of PSNR and SSIM.
235    for (int i = 0; i < num_dropped_from_prev_encode; i++) {
236      frame_writer_->WriteFrame(last_successful_frame_buffer_);
237    }
238  }
239  // Frame is not dropped, so update the encoded frame size
240  // (encoder callback is only called for non-zero length frames).
241  encoded_frame_size_ = encoded_image->_length;
242
243  TickTime encode_stop = TickTime::Now();
244  int frame_number = encoded_image->_timeStamp;
245  FrameStatistic& stat = stats_->stats_[frame_number];
246  stat.encode_time_in_us = GetElapsedTimeMicroseconds(encode_start_,
247                                                      encode_stop);
248  stat.encoding_successful = true;
249  stat.encoded_frame_length_in_bytes = encoded_image->_length;
250  stat.frame_number = encoded_image->_timeStamp;
251  stat.frame_type = encoded_image->_frameType;
252  stat.bit_rate_in_kbps = encoded_image->_length * bit_rate_factor_;
253  stat.total_packets = encoded_image->_length /
254      config_.networking_config.packet_size_in_bytes + 1;
255
256  // Perform packet loss if criteria is fullfilled:
257  bool exclude_this_frame = false;
258  // Only keyframes can be excluded
259  if (encoded_image->_frameType == kKeyFrame) {
260    switch (config_.exclude_frame_types) {
261      case kExcludeOnlyFirstKeyFrame:
262        if (!first_key_frame_has_been_excluded_) {
263          first_key_frame_has_been_excluded_ = true;
264          exclude_this_frame = true;
265        }
266        break;
267      case kExcludeAllKeyFrames:
268        exclude_this_frame = true;
269        break;
270      default:
271        assert(false);
272    }
273  }
274  if (!exclude_this_frame) {
275    stat.packets_dropped =
276          packet_manipulator_->ManipulatePackets(encoded_image);
277  }
278
279  // Keep track of if frames are lost due to packet loss so we can tell
280  // this to the encoder (this is handled by the RTP logic in the full stack)
281  decode_start_ = TickTime::Now();
282  // TODO(kjellander): Pass fragmentation header to the decoder when
283  // CL 172001 has been submitted and PacketManipulator supports this.
284  int32_t decode_result = decoder_->Decode(*encoded_image, last_frame_missing_,
285                                           NULL);
286  stat.decode_return_code = decode_result;
287  if (decode_result != WEBRTC_VIDEO_CODEC_OK) {
288    // Write the last successful frame the output file to avoid getting it out
289    // of sync with the source file for SSIM and PSNR comparisons:
290    frame_writer_->WriteFrame(last_successful_frame_buffer_);
291  }
292  // save status for losses so we can inform the decoder for the next frame:
293  last_frame_missing_ = encoded_image->_length == 0;
294}
295
296void VideoProcessorImpl::FrameDecoded(const I420VideoFrame& image) {
297  TickTime decode_stop = TickTime::Now();
298  int frame_number = image.timestamp();
299  // Report stats
300  FrameStatistic& stat = stats_->stats_[frame_number];
301  stat.decode_time_in_us = GetElapsedTimeMicroseconds(decode_start_,
302                                                      decode_stop);
303  stat.decoding_successful = true;
304
305  // Check for resize action (either down or up):
306  if (static_cast<int>(image.width()) != last_encoder_frame_width_ ||
307      static_cast<int>(image.height()) != last_encoder_frame_height_ ) {
308    ++num_spatial_resizes_;
309    last_encoder_frame_width_ = image.width();
310    last_encoder_frame_height_ = image.height();
311  }
312  // Check if codec size is different from native/original size, and if so,
313  // upsample back to original size: needed for PSNR and SSIM computations.
314  if (image.width() !=  config_.codec_settings->width ||
315      image.height() != config_.codec_settings->height) {
316    I420VideoFrame up_image;
317    int ret_val = scaler_.Set(image.width(), image.height(),
318                              config_.codec_settings->width,
319                              config_.codec_settings->height,
320                              kI420, kI420, kScaleBilinear);
321    assert(ret_val >= 0);
322    if (ret_val < 0) {
323      fprintf(stderr, "Failed to set scalar for frame: %d, return code: %d\n",
324              frame_number, ret_val);
325    }
326    ret_val = scaler_.Scale(image, &up_image);
327    assert(ret_val >= 0);
328    if (ret_val < 0) {
329      fprintf(stderr, "Failed to scale frame: %d, return code: %d\n",
330              frame_number, ret_val);
331    }
332    // TODO(mikhal): Extracting the buffer for now - need to update test.
333    int length = CalcBufferSize(kI420, up_image.width(), up_image.height());
334    scoped_ptr<uint8_t[]> image_buffer(new uint8_t[length]);
335    length = ExtractBuffer(up_image, length, image_buffer.get());
336    // Update our copy of the last successful frame:
337    memcpy(last_successful_frame_buffer_, image_buffer.get(), length);
338    bool write_success = frame_writer_->WriteFrame(image_buffer.get());
339    assert(write_success);
340    if (!write_success) {
341      fprintf(stderr, "Failed to write frame %d to disk!", frame_number);
342    }
343  } else {  // No resize.
344    // Update our copy of the last successful frame:
345    // TODO(mikhal): Add as a member function, so won't be allocated per frame.
346    int length = CalcBufferSize(kI420, image.width(), image.height());
347    scoped_ptr<uint8_t[]> image_buffer(new uint8_t[length]);
348    length = ExtractBuffer(image, length, image_buffer.get());
349    assert(length > 0);
350    memcpy(last_successful_frame_buffer_, image_buffer.get(), length);
351
352    bool write_success = frame_writer_->WriteFrame(image_buffer.get());
353    assert(write_success);
354    if (!write_success) {
355      fprintf(stderr, "Failed to write frame %d to disk!", frame_number);
356    }
357  }
358}
359
360int VideoProcessorImpl::GetElapsedTimeMicroseconds(
361    const webrtc::TickTime& start, const webrtc::TickTime& stop) {
362  uint64_t encode_time = (stop - start).Microseconds();
363  assert(encode_time <
364         static_cast<unsigned int>(std::numeric_limits<int>::max()));
365  return static_cast<int>(encode_time);
366}
367
368const char* ExcludeFrameTypesToStr(ExcludeFrameTypes e) {
369  switch (e) {
370    case kExcludeOnlyFirstKeyFrame:
371      return "ExcludeOnlyFirstKeyFrame";
372    case kExcludeAllKeyFrames:
373      return "ExcludeAllKeyFrames";
374    default:
375      assert(false);
376      return "Unknown";
377  }
378}
379
380const char* VideoCodecTypeToStr(webrtc::VideoCodecType e) {
381  switch (e) {
382    case kVideoCodecVP8:
383      return "VP8";
384    case kVideoCodecI420:
385      return "I420";
386    case kVideoCodecRED:
387      return "RED";
388    case kVideoCodecULPFEC:
389      return "ULPFEC";
390    case kVideoCodecUnknown:
391      return "Unknown";
392    default:
393      assert(false);
394      return "Unknown";
395  }
396}
397
398// Callbacks
399int32_t
400VideoProcessorImpl::VideoProcessorEncodeCompleteCallback::Encoded(
401    EncodedImage& encoded_image,
402    const webrtc::CodecSpecificInfo* codec_specific_info,
403    const webrtc::RTPFragmentationHeader* fragmentation) {
404  video_processor_->FrameEncoded(&encoded_image);  // Forward to parent class.
405  return 0;
406}
407int32_t
408VideoProcessorImpl::VideoProcessorDecodeCompleteCallback::Decoded(
409    I420VideoFrame& image) {
410  video_processor_->FrameDecoded(image);  // forward to parent class
411  return 0;
412}
413
414}  // namespace test
415}  // namespace webrtc
416