1/* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" 12 13#include <assert.h> 14#include <string.h> 15 16#include <limits> 17#include <vector> 18 19#include "webrtc/system_wrappers/interface/cpu_info.h" 20 21namespace webrtc { 22namespace test { 23 24TestConfig::TestConfig() 25 : name(""), 26 description(""), 27 test_number(0), 28 input_filename(""), 29 output_filename(""), 30 output_dir("out"), 31 networking_config(), 32 exclude_frame_types(kExcludeOnlyFirstKeyFrame), 33 frame_length_in_bytes(-1), 34 use_single_core(false), 35 keyframe_interval(0), 36 codec_settings(NULL), 37 verbose(true) {} 38 39TestConfig::~TestConfig() {} 40 41VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder, 42 webrtc::VideoDecoder* decoder, 43 FrameReader* frame_reader, 44 FrameWriter* frame_writer, 45 PacketManipulator* packet_manipulator, 46 const TestConfig& config, 47 Stats* stats) 48 : encoder_(encoder), 49 decoder_(decoder), 50 frame_reader_(frame_reader), 51 frame_writer_(frame_writer), 52 packet_manipulator_(packet_manipulator), 53 config_(config), 54 stats_(stats), 55 encode_callback_(NULL), 56 decode_callback_(NULL), 57 source_buffer_(NULL), 58 first_key_frame_has_been_excluded_(false), 59 last_frame_missing_(false), 60 initialized_(false), 61 encoded_frame_size_(0), 62 prev_time_stamp_(0), 63 num_dropped_frames_(0), 64 num_spatial_resizes_(0), 65 last_encoder_frame_width_(0), 66 last_encoder_frame_height_(0), 67 scaler_() { 68 assert(encoder); 69 assert(decoder); 70 assert(frame_reader); 71 assert(frame_writer); 72 assert(packet_manipulator); 73 assert(stats); 74} 75 76bool VideoProcessorImpl::Init() { 77 // Calculate a factor used for bit rate calculations: 78 bit_rate_factor_ = config_.codec_settings->maxFramerate * 0.001 * 8; // bits 79 80 // Initialize data structures used by the encoder/decoder APIs 81 size_t frame_length_in_bytes = frame_reader_->FrameLength(); 82 source_buffer_ = new uint8_t[frame_length_in_bytes]; 83 last_successful_frame_buffer_ = new uint8_t[frame_length_in_bytes]; 84 // Set fixed properties common for all frames. 85 // To keep track of spatial resize actions by encoder. 86 last_encoder_frame_width_ = config_.codec_settings->width; 87 last_encoder_frame_height_ = config_.codec_settings->height; 88 89 // Setup required callbacks for the encoder/decoder: 90 encode_callback_ = new VideoProcessorEncodeCompleteCallback(this); 91 decode_callback_ = new VideoProcessorDecodeCompleteCallback(this); 92 int32_t register_result = 93 encoder_->RegisterEncodeCompleteCallback(encode_callback_); 94 if (register_result != WEBRTC_VIDEO_CODEC_OK) { 95 fprintf(stderr, "Failed to register encode complete callback, return code: " 96 "%d\n", register_result); 97 return false; 98 } 99 register_result = decoder_->RegisterDecodeCompleteCallback(decode_callback_); 100 if (register_result != WEBRTC_VIDEO_CODEC_OK) { 101 fprintf(stderr, "Failed to register decode complete callback, return code: " 102 "%d\n", register_result); 103 return false; 104 } 105 // Init the encoder and decoder 106 uint32_t nbr_of_cores = 1; 107 if (!config_.use_single_core) { 108 nbr_of_cores = CpuInfo::DetectNumberOfCores(); 109 } 110 int32_t init_result = 111 encoder_->InitEncode(config_.codec_settings, nbr_of_cores, 112 config_.networking_config.max_payload_size_in_bytes); 113 if (init_result != WEBRTC_VIDEO_CODEC_OK) { 114 fprintf(stderr, "Failed to initialize VideoEncoder, return code: %d\n", 115 init_result); 116 return false; 117 } 118 init_result = decoder_->InitDecode(config_.codec_settings, nbr_of_cores); 119 if (init_result != WEBRTC_VIDEO_CODEC_OK) { 120 fprintf(stderr, "Failed to initialize VideoDecoder, return code: %d\n", 121 init_result); 122 return false; 123 } 124 125 if (config_.verbose) { 126 printf("Video Processor:\n"); 127 printf(" #CPU cores used : %d\n", nbr_of_cores); 128 printf(" Total # of frames: %d\n", frame_reader_->NumberOfFrames()); 129 printf(" Codec settings:\n"); 130 printf(" Start bitrate : %d kbps\n", 131 config_.codec_settings->startBitrate); 132 printf(" Width : %d\n", config_.codec_settings->width); 133 printf(" Height : %d\n", config_.codec_settings->height); 134 } 135 initialized_ = true; 136 return true; 137} 138 139VideoProcessorImpl::~VideoProcessorImpl() { 140 delete[] source_buffer_; 141 delete[] last_successful_frame_buffer_; 142 encoder_->RegisterEncodeCompleteCallback(NULL); 143 delete encode_callback_; 144 decoder_->RegisterDecodeCompleteCallback(NULL); 145 delete decode_callback_; 146} 147 148 149void VideoProcessorImpl::SetRates(int bit_rate, int frame_rate) { 150 int set_rates_result = encoder_->SetRates(bit_rate, frame_rate); 151 assert(set_rates_result >= 0); 152 if (set_rates_result < 0) { 153 fprintf(stderr, "Failed to update encoder with new rate %d, " 154 "return code: %d\n", bit_rate, set_rates_result); 155 } 156 num_dropped_frames_ = 0; 157 num_spatial_resizes_ = 0; 158} 159 160int VideoProcessorImpl::EncodedFrameSize() { 161 return encoded_frame_size_; 162} 163 164int VideoProcessorImpl::NumberDroppedFrames() { 165 return num_dropped_frames_; 166} 167 168int VideoProcessorImpl::NumberSpatialResizes() { 169 return num_spatial_resizes_; 170} 171 172bool VideoProcessorImpl::ProcessFrame(int frame_number) { 173 assert(frame_number >=0); 174 if (!initialized_) { 175 fprintf(stderr, "Attempting to use uninitialized VideoProcessor!\n"); 176 return false; 177 } 178 // |prev_time_stamp_| is used for getting number of dropped frames. 179 if (frame_number == 0) { 180 prev_time_stamp_ = -1; 181 } 182 if (frame_reader_->ReadFrame(source_buffer_)) { 183 // Copy the source frame to the newly read frame data. 184 int size_y = config_.codec_settings->width * config_.codec_settings->height; 185 int half_width = (config_.codec_settings->width + 1) / 2; 186 int half_height = (config_.codec_settings->height + 1) / 2; 187 int size_uv = half_width * half_height; 188 source_frame_.CreateFrame(size_y, source_buffer_, 189 size_uv, source_buffer_ + size_y, 190 size_uv, source_buffer_ + size_y + size_uv, 191 config_.codec_settings->width, 192 config_.codec_settings->height, 193 config_.codec_settings->width, 194 half_width, half_width); 195 196 // Ensure we have a new statistics data object we can fill: 197 FrameStatistic& stat = stats_->NewFrame(frame_number); 198 199 encode_start_ = TickTime::Now(); 200 // Use the frame number as "timestamp" to identify frames 201 source_frame_.set_timestamp(frame_number); 202 203 // Decide if we're going to force a keyframe: 204 std::vector<VideoFrameType> frame_types(1, kDeltaFrame); 205 if (config_.keyframe_interval > 0 && 206 frame_number % config_.keyframe_interval == 0) { 207 frame_types[0] = kKeyFrame; 208 } 209 210 // For dropped frames, we regard them as zero size encoded frames. 211 encoded_frame_size_ = 0; 212 213 int32_t encode_result = encoder_->Encode(source_frame_, NULL, &frame_types); 214 215 if (encode_result != WEBRTC_VIDEO_CODEC_OK) { 216 fprintf(stderr, "Failed to encode frame %d, return code: %d\n", 217 frame_number, encode_result); 218 } 219 stat.encode_return_code = encode_result; 220 return true; 221 } else { 222 return false; // we've reached the last frame 223 } 224} 225 226void VideoProcessorImpl::FrameEncoded(EncodedImage* encoded_image) { 227 // Timestamp is frame number, so this gives us #dropped frames. 228 int num_dropped_from_prev_encode = encoded_image->_timeStamp - 229 prev_time_stamp_ - 1; 230 num_dropped_frames_ += num_dropped_from_prev_encode; 231 prev_time_stamp_ = encoded_image->_timeStamp; 232 if (num_dropped_from_prev_encode > 0) { 233 // For dropped frames, we write out the last decoded frame to avoid getting 234 // out of sync for the computation of PSNR and SSIM. 235 for (int i = 0; i < num_dropped_from_prev_encode; i++) { 236 frame_writer_->WriteFrame(last_successful_frame_buffer_); 237 } 238 } 239 // Frame is not dropped, so update the encoded frame size 240 // (encoder callback is only called for non-zero length frames). 241 encoded_frame_size_ = encoded_image->_length; 242 243 TickTime encode_stop = TickTime::Now(); 244 int frame_number = encoded_image->_timeStamp; 245 FrameStatistic& stat = stats_->stats_[frame_number]; 246 stat.encode_time_in_us = GetElapsedTimeMicroseconds(encode_start_, 247 encode_stop); 248 stat.encoding_successful = true; 249 stat.encoded_frame_length_in_bytes = encoded_image->_length; 250 stat.frame_number = encoded_image->_timeStamp; 251 stat.frame_type = encoded_image->_frameType; 252 stat.bit_rate_in_kbps = encoded_image->_length * bit_rate_factor_; 253 stat.total_packets = encoded_image->_length / 254 config_.networking_config.packet_size_in_bytes + 1; 255 256 // Perform packet loss if criteria is fullfilled: 257 bool exclude_this_frame = false; 258 // Only keyframes can be excluded 259 if (encoded_image->_frameType == kKeyFrame) { 260 switch (config_.exclude_frame_types) { 261 case kExcludeOnlyFirstKeyFrame: 262 if (!first_key_frame_has_been_excluded_) { 263 first_key_frame_has_been_excluded_ = true; 264 exclude_this_frame = true; 265 } 266 break; 267 case kExcludeAllKeyFrames: 268 exclude_this_frame = true; 269 break; 270 default: 271 assert(false); 272 } 273 } 274 if (!exclude_this_frame) { 275 stat.packets_dropped = 276 packet_manipulator_->ManipulatePackets(encoded_image); 277 } 278 279 // Keep track of if frames are lost due to packet loss so we can tell 280 // this to the encoder (this is handled by the RTP logic in the full stack) 281 decode_start_ = TickTime::Now(); 282 // TODO(kjellander): Pass fragmentation header to the decoder when 283 // CL 172001 has been submitted and PacketManipulator supports this. 284 int32_t decode_result = decoder_->Decode(*encoded_image, last_frame_missing_, 285 NULL); 286 stat.decode_return_code = decode_result; 287 if (decode_result != WEBRTC_VIDEO_CODEC_OK) { 288 // Write the last successful frame the output file to avoid getting it out 289 // of sync with the source file for SSIM and PSNR comparisons: 290 frame_writer_->WriteFrame(last_successful_frame_buffer_); 291 } 292 // save status for losses so we can inform the decoder for the next frame: 293 last_frame_missing_ = encoded_image->_length == 0; 294} 295 296void VideoProcessorImpl::FrameDecoded(const I420VideoFrame& image) { 297 TickTime decode_stop = TickTime::Now(); 298 int frame_number = image.timestamp(); 299 // Report stats 300 FrameStatistic& stat = stats_->stats_[frame_number]; 301 stat.decode_time_in_us = GetElapsedTimeMicroseconds(decode_start_, 302 decode_stop); 303 stat.decoding_successful = true; 304 305 // Check for resize action (either down or up): 306 if (static_cast<int>(image.width()) != last_encoder_frame_width_ || 307 static_cast<int>(image.height()) != last_encoder_frame_height_ ) { 308 ++num_spatial_resizes_; 309 last_encoder_frame_width_ = image.width(); 310 last_encoder_frame_height_ = image.height(); 311 } 312 // Check if codec size is different from native/original size, and if so, 313 // upsample back to original size: needed for PSNR and SSIM computations. 314 if (image.width() != config_.codec_settings->width || 315 image.height() != config_.codec_settings->height) { 316 I420VideoFrame up_image; 317 int ret_val = scaler_.Set(image.width(), image.height(), 318 config_.codec_settings->width, 319 config_.codec_settings->height, 320 kI420, kI420, kScaleBilinear); 321 assert(ret_val >= 0); 322 if (ret_val < 0) { 323 fprintf(stderr, "Failed to set scalar for frame: %d, return code: %d\n", 324 frame_number, ret_val); 325 } 326 ret_val = scaler_.Scale(image, &up_image); 327 assert(ret_val >= 0); 328 if (ret_val < 0) { 329 fprintf(stderr, "Failed to scale frame: %d, return code: %d\n", 330 frame_number, ret_val); 331 } 332 // TODO(mikhal): Extracting the buffer for now - need to update test. 333 int length = CalcBufferSize(kI420, up_image.width(), up_image.height()); 334 scoped_ptr<uint8_t[]> image_buffer(new uint8_t[length]); 335 length = ExtractBuffer(up_image, length, image_buffer.get()); 336 // Update our copy of the last successful frame: 337 memcpy(last_successful_frame_buffer_, image_buffer.get(), length); 338 bool write_success = frame_writer_->WriteFrame(image_buffer.get()); 339 assert(write_success); 340 if (!write_success) { 341 fprintf(stderr, "Failed to write frame %d to disk!", frame_number); 342 } 343 } else { // No resize. 344 // Update our copy of the last successful frame: 345 // TODO(mikhal): Add as a member function, so won't be allocated per frame. 346 int length = CalcBufferSize(kI420, image.width(), image.height()); 347 scoped_ptr<uint8_t[]> image_buffer(new uint8_t[length]); 348 length = ExtractBuffer(image, length, image_buffer.get()); 349 assert(length > 0); 350 memcpy(last_successful_frame_buffer_, image_buffer.get(), length); 351 352 bool write_success = frame_writer_->WriteFrame(image_buffer.get()); 353 assert(write_success); 354 if (!write_success) { 355 fprintf(stderr, "Failed to write frame %d to disk!", frame_number); 356 } 357 } 358} 359 360int VideoProcessorImpl::GetElapsedTimeMicroseconds( 361 const webrtc::TickTime& start, const webrtc::TickTime& stop) { 362 uint64_t encode_time = (stop - start).Microseconds(); 363 assert(encode_time < 364 static_cast<unsigned int>(std::numeric_limits<int>::max())); 365 return static_cast<int>(encode_time); 366} 367 368const char* ExcludeFrameTypesToStr(ExcludeFrameTypes e) { 369 switch (e) { 370 case kExcludeOnlyFirstKeyFrame: 371 return "ExcludeOnlyFirstKeyFrame"; 372 case kExcludeAllKeyFrames: 373 return "ExcludeAllKeyFrames"; 374 default: 375 assert(false); 376 return "Unknown"; 377 } 378} 379 380const char* VideoCodecTypeToStr(webrtc::VideoCodecType e) { 381 switch (e) { 382 case kVideoCodecVP8: 383 return "VP8"; 384 case kVideoCodecI420: 385 return "I420"; 386 case kVideoCodecRED: 387 return "RED"; 388 case kVideoCodecULPFEC: 389 return "ULPFEC"; 390 case kVideoCodecUnknown: 391 return "Unknown"; 392 default: 393 assert(false); 394 return "Unknown"; 395 } 396} 397 398// Callbacks 399int32_t 400VideoProcessorImpl::VideoProcessorEncodeCompleteCallback::Encoded( 401 EncodedImage& encoded_image, 402 const webrtc::CodecSpecificInfo* codec_specific_info, 403 const webrtc::RTPFragmentationHeader* fragmentation) { 404 video_processor_->FrameEncoded(&encoded_image); // Forward to parent class. 405 return 0; 406} 407int32_t 408VideoProcessorImpl::VideoProcessorDecodeCompleteCallback::Decoded( 409 I420VideoFrame& image) { 410 video_processor_->FrameDecoded(image); // forward to parent class 411 return 0; 412} 413 414} // namespace test 415} // namespace webrtc 416